diff --git a/README.md b/README.md index 8c26143a12..3fbd166f49 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,7 @@ The TDengine community has also kindly built some of their own connectors! Follo - [Rust Connector](https://github.com/taosdata/TDengine/tree/master/tests/examples/rust) - [.Net Core Connector](https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos) +- [Lua Connector](https://github.com/taosdata/TDengine/tree/develop/tests/examples/lua) # How to run the test cases and how to add a new test case? TDengine's test framework and all test cases are fully open source. diff --git a/cmake/install.inc b/cmake/install.inc index 8016f23808..7a92a396e3 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -2,7 +2,7 @@ IF (TD_LINUX) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})") - INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR})") + INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR} Linux ${TD_VER_NUMBER})") ELSEIF (TD_WINDOWS) IF (TD_POWER) SET(CMAKE_INSTALL_PREFIX C:/PowerDB) @@ -19,8 +19,14 @@ ELSEIF (TD_WINDOWS) INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) - INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) + + IF (TD_POWER) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/power.exe DESTINATION .) + ELSE () + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .) + ENDIF () + #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) @@ -34,5 +40,5 @@ ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})") - INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR} Darwin)") + INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR} Darwin ${TD_VER_NUMBER})") ENDIF () diff --git a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md index b78200be04..b0f8ed276d 100644 --- a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md @@ -416,7 +416,7 @@ taos> SELECT database(); power | Query OK, 1 row(s) in set (0.000079s) ``` -如果登录的时候没有指定默认数据库，且没有使用```use``命令切换数据，则返回NULL。 +如果登录的时候没有指定默认数据库，且没有使用```use```命令切换数据，则返回NULL。 ``` taos> SELECT database(); database() | @@ -503,10 +503,10 @@ Query OK, 1 row(s) in set (0.001091s) | % | match with any char sequences | **`binary`** **`nchar`** | | _ | match with a single char | **`binary`** **`nchar`** | -1. 同时进行多个字段的范围过滤需要使用关键词AND进行连接不同的查询条件，暂不支持OR连接的查询条件。 -2. 针对某一字段的过滤只支持单一区间的过滤条件。例如：value>20 and value<30是合法的过滤条件, 而Value<20 AND value<>5是非法的过滤条件。 +1. 同时进行多个字段的范围过滤需要使用关键词AND进行连接不同的查询条件，暂不支持OR连接的不同列之间的查询过滤条件。 +2. 针对某一字段的过滤只支持单一时间区间过滤条件。但是针对其他的（普通）列或标签列，可以使用``` OR``` 条件进行组合条件的查询过滤。例如：((value > 20 and value < 30) OR (value < 12)) 。 -### Some Examples +### SQL 示例 - 对于下面的例子，表tb1用以下语句创建 @@ -538,7 +538,7 @@ Query OK, 1 row(s) in set (0.001091s) SELECT COUNT(*) FROM tb1 WHERE ts >= NOW - 10m AND col2 > 3.14 >> /home/testoutpu.csv; ``` -## SQL函数 +## SQL 函数 ### 聚合函数 diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 266aea4c51..27c1054dc8 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -59,36 +59,52 @@ -n ：指示运行网络连通检测的服务端功能，或客户端功能，缺省值为空，表示不启动网络连通检测； - -h：指示服务端名称，可以是ip地址或fqdn格式。如：192.168.1.160，或 192.168.1.160:6030，或 hostname1，或hostname1:6030。缺省值是127.0.01。 + -h：指示服务端名称，可以是ip地址或fqdn格式。如：192.168.1.160，或 192.168.1.160:6030，或 hostname1，或hostname1:6030。缺省值是127.0.0.1。 -P ：检测的起始端口号，缺省值是6030； -e：检测的结束端口号，必须大于等于起始端口号，缺省值是6042； - -l：指定检测端口连通的报文长度，最大64000字节，缺省值是1000字节； + -l：指定检测端口连通的报文长度，最大64000字节，缺省值是1000字节，测试时服务端和客户端必须指定相同； 服务端设置的起始端口和结束端口号，必须包含客户端设置的起始端口和结束端口号； - 对于客户端，起始端口号的有三种方式：缺省值、-h指定、-P指定，优先级是：-P指定 > -h指定 > 缺省值。 + 对于起始端口号有三种设置方式：缺省值、-h指定、-P指定，优先级是：-P指定 > -h指定 > 缺省值。 客户端运行的输出样例： - `sum@sum-virtualBox /home/sum $ taos -n client -h ubuntu-vbox6 - host: ubuntu-vbox6 start port: 6030 end port: 6042 packet len: 1000 + `sum@sum-virtualBox /home/sum $ taos -n client -h ubuntu-vbox6` + + `host: ubuntu-vbox6 start port: 6030 end port: 6042 packet len: 1000` + + `tcp port:6030 test ok. udp port:6030 test ok.` + + `tcp port:6031 test ok. udp port:6031 test ok.` + + `tcp port:6032 test ok. udp port:6032 test ok.` + + `tcp port:6033 test ok. udp port:6033 test ok.` + + `tcp port:6034 test ok. udp port:6034 test ok.` + + `tcp port:6035 test ok. udp port:6035 test ok.` + + `tcp port:6036 test ok. udp port:6036 test ok.` + + `tcp port:6037 test ok. udp port:6037 test ok.` + + `tcp port:6038 test ok. udp port:6038 test ok.` + + `tcp port:6039 test ok. udp port:6039 test ok.` + + `tcp port:6040 test ok. udp port:6040 test ok.` + + `tcp port:6041 test ok. udp port:6041 test ok.` + + `tcp port:6042 test ok. udp port:6042 test ok.` + + 如果某个端口不通，会输出 `port：xxxx test fail`的信息。 - tcp port:6030 test ok. udp port:6030 test ok. - tcp port:6031 test ok. udp port:6031 test ok. - tcp port:6032 test ok. udp port:6032 test ok. - tcp port:6033 test ok. udp port:6033 test ok. - tcp port:6034 test ok. udp port:6034 test ok. - tcp port:6035 test ok. udp port:6035 test ok. - tcp port:6036 test ok. udp port:6036 test ok. - tcp port:6037 test ok. udp port:6037 test ok. - tcp port:6038 test ok. udp port:6038 test ok. - tcp port:6039 test ok. udp port:6039 test ok. - tcp port:6040 test ok. udp port:6040 test ok. - tcp port:6041 test ok. udp port:6041 test ok. - tcp port:6042 test ok. udp port:6042 test ok.` ## 6. 遇到错误“Unexpected generic error in RPC”， 我怎么办？ 产生这个错误，是由于客户端或数据节点无法解析FQDN(Fully Qualified Domain Name)导致。对于TAOS Shell或客户端应用，请做如下检查： diff --git a/packaging/tools/install_arbi_power.sh b/packaging/tools/install_arbi_power.sh old mode 100644 new mode 100755 diff --git a/packaging/tools/install_client_power.sh b/packaging/tools/install_client_power.sh old mode 100644 new mode 100755 diff --git a/packaging/tools/install_power.sh b/packaging/tools/install_power.sh old mode 100644 new mode 100755 diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 74aa1495fe..eff70d8035 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -10,6 +10,7 @@ set -e source_dir=$1 binary_dir=$2 osType=$3 +verNumber=$4 if [ "$osType" != "Darwin" ]; then script_dir=$(dirname $(readlink -f "$0")) @@ -179,19 +180,18 @@ function install_lib() { ${csudo} rm -f ${lib_link_dir}/libtaos.* || : ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : - versioninfo=$(${script_dir}/get_version.sh ${source_dir}/src/util/src/version.c) if [ "$osType" != "Darwin" ]; then - ${csudo} cp ${binary_dir}/build/lib/libtaos.so.${versioninfo} ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* - ${csudo} ln -sf ${install_main_dir}/driver/libtaos.so.${versioninfo} ${lib_link_dir}/libtaos.so.1 + ${csudo} cp ${binary_dir}/build/lib/libtaos.so.${verNumber} ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + ${csudo} ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so if [ -d "${lib64_link_dir}" ]; then - ${csudo} ln -sf ${install_main_dir}/driver/libtaos.so.${versioninfo} ${lib64_link_dir}/libtaos.so.1 + ${csudo} ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 ${csudo} ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so fi else - ${csudo} cp ${binary_dir}/build/lib/libtaos.${versioninfo}.dylib ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* - ${csudo} ln -sf ${install_main_dir}/driver/libtaos.${versioninfo}.dylib ${lib_link_dir}/libtaos.1.dylib + ${csudo} cp ${binary_dir}/build/lib/libtaos.* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + ${csudo} ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib ${csudo} ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib fi diff --git a/packaging/tools/makearbi_power.sh b/packaging/tools/makearbi_power.sh old mode 100644 new mode 100755 diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh old mode 100644 new mode 100755 diff --git a/packaging/tools/makepkg_power.sh b/packaging/tools/makepkg_power.sh old mode 100644 new mode 100755 diff --git a/packaging/tools/remove_arbi_power.sh b/packaging/tools/remove_arbi_power.sh old mode 100644 new mode 100755 diff --git a/packaging/tools/remove_client_power.sh b/packaging/tools/remove_client_power.sh old mode 100644 new mode 100755 diff --git a/packaging/tools/remove_power.sh b/packaging/tools/remove_power.sh old mode 100644 new mode 100755 diff --git a/snap/hooks/install b/snap/hooks/install index e58918d2c3..542be0b834 100755 --- a/snap/hooks/install +++ b/snap/hooks/install @@ -5,7 +5,9 @@ if [ ! -d /var/lib/taos ]; then fi if [ ! -d /var/log/taos ]; then - mkdir -p -m777 /var/log/taos + mkdir -p --mode=777 /var/log/taos +else + chmod 777 /var/log/taos fi if [ ! -d /etc/taos ]; then @@ -13,5 +15,8 @@ if [ ! -d /etc/taos ]; then fi if [ ! -f /etc/taos/taos.cfg ]; then + if [ ! -d /etc/taos ]; then + mkdir -p /etc/taos + fi cp $SNAP/etc/taos/taos.cfg /etc/taos/taos.cfg fi diff --git a/snap/local/launcher.sh b/snap/local/launcher.sh index 52b3e4ce5c..29a7a63779 100755 --- a/snap/local/launcher.sh +++ b/snap/local/launcher.sh @@ -15,11 +15,12 @@ case "$SNAP_USER_COMMON" in *) COMMON=$SNAP_USER_COMMON ;; esac -if [ -d /etc/taos ]; then - CONFIG_FILE="/etc/taos" -else - CONFIG_FILE="$SNAP/etc/taos" +if [ ! -f $SNAP_DATA/etc/taos/taos.cfg ]; then + if [ ! -d $SNAP_DATA/etc/taos ]; then + mkdir -p $SNAP_DATA/etc/taos + fi + cp $SNAP/etc/taos/taos.cfg $SNAP_DATA/etc/taos fi # Launch the snap -$SNAP/usr/bin/taosd -c $CONFIG_FILE $@ +$SNAP/usr/bin/taosd -c /etc/taos $@ diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index cf4f993394..7a0e1c3b80 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,13 +1,13 @@ name: tdengine base: core18 # the base snap is the execution environment for this snap -version: '2.0.0.6' # just for humans, typically '1.2+git' or '1.3.2' +version: '2.0.2.0' # just for humans, typically '1.2+git' or '1.3.2' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | TDengine is an open-source big data platform designed and optimized for Internet of Things (IoT), Connected Vehicles, and Industrial IoT. Besides the 10x faster time-series database, it provides caching, stream computing, message queuing and other functionalities to reduce the complexity and costs of development and operations. grade: stable -confinement: classic +confinement: strict apps: tdengine: @@ -24,7 +24,9 @@ apps: command: taoswrapper.sh plugs: - network + - system-observe - systemfiles + - historyfile taosdemo: command: usr/bin/taosdemo @@ -32,11 +34,19 @@ apps: - network plugs: + historyfile: + interface: personal-files + read: + - $HOME/.taos_history + write: + - $HOME/.taos_history + systemfiles: interface: system-files read: - /etc/taos - /var/lib/taos + - /var/log/taos - /tmp write: - /var/log/taos @@ -77,7 +87,7 @@ parts: mkdir -p $SNAPCRAFT_STAGE/var/lib/taos fi if [ ! -d $SNAPCRAFT_STAGE/var/log/taos ]; then - mkdir -p $SNAPCRAFT_STAGE/var/log/taos + mkdir -p --mode=777 $SNAPCRAFT_STAGE/var/log/taos fi prime: @@ -85,16 +95,16 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.0.6 + - usr/lib/libtaos.so.2.0.2.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so override-prime: | snapcraftctl prime - if [ ! -d $SNAPCRAFT_STAGE/var/lib/taos ]; then + if [ ! -d $SNAPCRAFT_PRIME/var/lib/taos ]; then cp -rf $SNAPCRAFT_STAGE/var/lib/taos $SNAPCRAFT_PRIME fi - if [ ! -d $SNAPCRAFT_STAGE/var/log/taos ]; then + if [ ! -d $SNAPCRAFT_PRIME/var/log/taos ]; then cp -rf $SNAPCRAFT_STAGE/var/log/taos $SNAPCRAFT_PRIME fi @@ -103,11 +113,10 @@ layout: bind: $SNAP_DATA/var/lib/taos /var/log/taos: bind: $SNAP_DATA/var/log/taos - /etc/taos/taos.cfg: - bind-file: $SNAP_DATA/etc/taos/taos.cfg + /etc/taos: + bind: $SNAP_DATA/etc/taos hooks: install: - plugs: [systemfiles] - + plugs: [systemfiles, historyfile] diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 5b5fb3435d..daf7c5e534 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -23,12 +23,8 @@ IF (TD_LINUX) #set version of .so #VERSION so version #SOVERSION api version - execute_process(COMMAND chmod 777 ${TD_COMMUNITY_DIR}/packaging/tools/get_version.sh) - execute_process(COMMAND ${TD_COMMUNITY_DIR}/packaging/tools/get_version.sh ${TD_COMMUNITY_DIR}/src/util/src/version.c - OUTPUT_VARIABLE - VERSION_INFO) - MESSAGE(STATUS "build version ${VERSION_INFO}") - SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${VERSION_INFO} SOVERSION 1) + #MESSAGE(STATUS "build version ${TD_VER_NUMBER}") + SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1) ADD_SUBDIRECTORY(tests) @@ -65,11 +61,7 @@ ELSEIF (TD_DARWIN) #set version of .so #VERSION so version #SOVERSION api version - execute_process(COMMAND chmod 777 ${TD_COMMUNITY_DIR}/packaging/tools/get_version.sh) - execute_process(COMMAND ${TD_COMMUNITY_DIR}/packaging/tools/get_version.sh ${TD_COMMUNITY_DIR}/src/util/src/version.c - OUTPUT_VARIABLE - VERSION_INFO) - MESSAGE(STATUS "build version ${VERSION_INFO}") - SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${VERSION_INFO} SOVERSION 1) + #MESSAGE(STATUS "build version ${TD_VER_NUMBER}") + SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1) ENDIF () diff --git a/src/client/inc/tscSubquery.h b/src/client/inc/tscSubquery.h index d5833675aa..07e0580397 100644 --- a/src/client/inc/tscSubquery.h +++ b/src/client/inc/tscSubquery.h @@ -30,7 +30,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code); SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, int32_t index); -int32_t tscHandleMasterJoinQuery(SSqlObj* pSql); +void tscHandleMasterJoinQuery(SSqlObj* pSql); int32_t tscHandleMasterSTableQuery(SSqlObj *pSql); diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 46a576fa9a..9b31b8fc6a 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -39,7 +39,6 @@ extern "C" { #define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo)\ (!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo))) -#define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) typedef struct SParsedColElem { int16_t colIndex; @@ -188,7 +187,7 @@ SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functi size_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo); SSqlExpr* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index); -void tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); +int32_t tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); void tscSqlExprInfoDestroy(SArray* pExprInfo); SColumn* tscColumnClone(const SColumn* src); @@ -206,7 +205,7 @@ bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid); void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw); -void tscTagCondCopy(STagCond* dest, const STagCond* src); +int32_t tscTagCondCopy(STagCond* dest, const STagCond* src); void tscTagCondRelease(STagCond* pCond); void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 900d4955e9..57a4cb29c1 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -458,6 +458,7 @@ bool tscResultsetFetchCompleted(TAOS_RES *result); char *tscGetErrorMsgPayload(SSqlCmd *pCmd); int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql); +int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* sql); int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); @@ -471,7 +472,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row; // user defined constant value output columns - if (pInfo->pSqlExpr->colInfo.flag == TSDB_COL_UDC) { + if (TSDB_COL_IS_UD_COL(pInfo->pSqlExpr->colInfo.flag)) { if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { pData = pInfo->pSqlExpr->param[1].pz; pRes->length[columnIndex] = pInfo->pSqlExpr->param[1].nLen; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 4643d255dc..41aa122160 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -50,7 +50,8 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const pSql->sqlstr = calloc(1, sqlLen + 1); if (pSql->sqlstr == NULL) { tscError("%p failed to malloc sql string buffer", pSql); - tscQueueAsyncError(pSql->fp, pSql->param, TSDB_CODE_TSC_OUT_OF_MEMORY); + pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; + tscQueueAsyncRes(pSql); return; } @@ -94,7 +95,6 @@ void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *pa SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); if (pSql == NULL) { tscError("failed to malloc sqlObj"); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tscQueueAsyncError(fp, param, TSDB_CODE_TSC_OUT_OF_MEMORY); return; } @@ -191,7 +191,7 @@ void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRo tscProcessAsyncRetrieveImpl(param, tres, numOfRows, tscAsyncFetchSingleRowProxy); } -void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), void *param) { +void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) { SSqlObj *pSql = (SSqlObj *)taosa; if (pSql == NULL || pSql->signature != pSql) { tscError("sql object is NULL"); @@ -209,6 +209,8 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi if (pRes->qhandle == 0) { tscError("qhandle is NULL"); pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; + pSql->param = param; + tscQueueAsyncRes(pSql); return; } @@ -269,7 +271,10 @@ void taos_fetch_row_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, TAOS_ROW), if (pRes->qhandle == 0) { tscError("qhandle is NULL"); - tscQueueAsyncError(fp, param, TSDB_CODE_TSC_INVALID_QHANDLE); + pSql->param = param; + pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; + + tscQueueAsyncRes(pSql); return; } @@ -352,36 +357,17 @@ void tscProcessFetchRow(SSchedMsg *pMsg) { void tscProcessAsyncRes(SSchedMsg *pMsg) { SSqlObj *pSql = (SSqlObj *)pMsg->ahandle; -// SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - -// void *taosres = pSql; - - // pCmd may be released, so cache pCmd->command -// int cmd = pCmd->command; -// int code = pRes->code; - - // in case of async insert, restore the user specified callback function -// bool shouldFree = tscShouldBeFreed(pSql); - -// if (pCmd->command == TSDB_SQL_INSERT) { -// assert(pSql->fp != NULL); assert(pSql->fp != NULL && pSql->fetchFp != NULL); -// } -// if (pSql->fp) { pSql->fp = pSql->fetchFp; (*pSql->fp)(pSql->param, pSql, pRes->code); -// } - -// if (shouldFree) { -// tscDebug("%p sqlObj is automatically freed in async res", pSql); -// tscFreeSqlObj(pSql); -// } } +// this function will be executed by queue task threads, so the terrno is not valid static void tscProcessAsyncError(SSchedMsg *pMsg) { void (*fp)() = pMsg->ahandle; + terrno = *(int32_t*) pMsg->msg; (*fp)(pMsg->thandle, NULL, *(int32_t*)pMsg->msg); } diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index e74fcba246..4b31a8001f 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -1648,9 +1648,10 @@ static void last_function(SQLFunctionCtx *pCtx) { for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; + if (!pCtx->requireNull) { + continue; + } } - memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); TSKEY ts = pCtx->ptsList[i]; @@ -1721,7 +1722,9 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; + if (!pCtx->requireNull) { + continue; + } } last_data_assign_impl(pCtx, data, i); @@ -2422,24 +2425,14 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { /////////////////////////////////////////////////////////////////////////////////////////////// static bool percentile_function_setup(SQLFunctionCtx *pCtx) { - const int32_t MAX_AVAILABLE_BUFFER_SIZE = 1 << 20; // 1MB - const int32_t NUMOFCOLS = 1; - if (!function_setup(pCtx)) { return false; } SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SSchema field[1] = { { (uint8_t)pCtx->inputType, "dummyCol", 0, pCtx->inputBytes } }; - - SColumnModel *pModel = createColumnModel(field, 1, 1000); - int32_t orderIdx = 0; - - // tOrderDesc object - tOrderDescriptor *pDesc = tOrderDesCreate(&orderIdx, NUMOFCOLS, pModel, TSDB_ORDER_DESC); - + ((SPercentileInfo *)(pResInfo->interResultBuf))->pMemBucket = - tMemBucketCreate(1024, MAX_AVAILABLE_BUFFER_SIZE, pCtx->inputBytes, pCtx->inputType, pDesc); + tMemBucketCreate(pCtx->inputBytes, pCtx->inputType); return true; } @@ -2485,15 +2478,13 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); tMemBucket * pMemBucket = ((SPercentileInfo *)pResInfo->interResultBuf)->pMemBucket; - if (pMemBucket->numOfElems > 0) { // check for null + if (pMemBucket->total > 0) { // check for null *(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v); } else { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); } - tOrderDescDestroy(pMemBucket->pOrderDesc); tMemBucketDestroy(pMemBucket); - doFinalizer(pCtx); } diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index caaaa5bc18..b240d357a8 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -274,7 +274,7 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) { return tscSetValueToResObj(pSql, rowLen); } -static void tscProcessCurrentUser(SSqlObj *pSql) { +static int32_t tscProcessCurrentUser(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); @@ -282,14 +282,20 @@ static void tscProcessCurrentUser(SSqlObj *pSql) { pExpr->resType = TSDB_DATA_TYPE_BINARY; char* vx = calloc(1, pExpr->resBytes); + if (vx == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + size_t size = sizeof(pSql->pTscObj->user); STR_WITH_MAXSIZE_TO_VARSTR(vx, pSql->pTscObj->user, size); tscSetLocalQueryResult(pSql, vx, pExpr->aliasName, pExpr->resType, pExpr->resBytes); free(vx); + + return TSDB_CODE_SUCCESS; } -static void tscProcessCurrentDB(SSqlObj *pSql) { +static int32_t tscProcessCurrentDB(SSqlObj *pSql) { char db[TSDB_DB_NAME_LEN] = {0}; extractDBName(pSql->pTscObj->db, db); @@ -302,6 +308,10 @@ static void tscProcessCurrentDB(SSqlObj *pSql) { pExpr->resBytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE; char* vx = calloc(1, pExpr->resBytes); + if (vx == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + if (t == 0) { setVardataNull(vx, TSDB_DATA_TYPE_BINARY); } else { @@ -310,9 +320,11 @@ static void tscProcessCurrentDB(SSqlObj *pSql) { tscSetLocalQueryResult(pSql, vx, pExpr->aliasName, pExpr->resType, pExpr->resBytes); free(vx); + + return TSDB_CODE_SUCCESS; } -static void tscProcessServerVer(SSqlObj *pSql) { +static int32_t tscProcessServerVer(SSqlObj *pSql) { const char* v = pSql->pTscObj->sversion; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); @@ -323,13 +335,18 @@ static void tscProcessServerVer(SSqlObj *pSql) { pExpr->resBytes = (int16_t)(t + VARSTR_HEADER_SIZE); char* vx = calloc(1, pExpr->resBytes); + if (vx == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + STR_WITH_SIZE_TO_VARSTR(vx, v, (VarDataLenT)t); tscSetLocalQueryResult(pSql, vx, pExpr->aliasName, pExpr->resType, pExpr->resBytes); - taosTFree(vx); + free(vx); + return TSDB_CODE_SUCCESS; } -static void tscProcessClientVer(SSqlObj *pSql) { +static int32_t tscProcessClientVer(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); @@ -339,23 +356,28 @@ static void tscProcessClientVer(SSqlObj *pSql) { pExpr->resBytes = (int16_t)(t + VARSTR_HEADER_SIZE); char* v = calloc(1, pExpr->resBytes); + if (v == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + STR_WITH_SIZE_TO_VARSTR(v, version, (VarDataLenT)t); tscSetLocalQueryResult(pSql, v, pExpr->aliasName, pExpr->resType, pExpr->resBytes); - taosTFree(v); + free(v); + return TSDB_CODE_SUCCESS; } -static void tscProcessServStatus(SSqlObj *pSql) { +static int32_t tscProcessServStatus(SSqlObj *pSql) { STscObj* pObj = pSql->pTscObj; if (pObj->pHb != NULL) { if (pObj->pHb->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - return; + return pSql->res.code; } } else { if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - return; + return pSql->res.code; } } @@ -364,6 +386,7 @@ static void tscProcessServStatus(SSqlObj *pSql) { SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); int32_t val = 1; tscSetLocalQueryResult(pSql, (char*) &val, pExpr->aliasName, TSDB_DATA_TYPE_INT, sizeof(int32_t)); + return TSDB_CODE_SUCCESS; } void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength) { @@ -393,37 +416,39 @@ void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnNa int tscProcessLocalCmd(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; + SSqlRes *pRes = &pSql->res; if (pCmd->command == TSDB_SQL_CFG_LOCAL) { - pSql->res.code = (uint8_t)taosCfgDynamicOptions(pCmd->payload); + pRes->code = (uint8_t)taosCfgDynamicOptions(pCmd->payload); } else if (pCmd->command == TSDB_SQL_DESCRIBE_TABLE) { - pSql->res.code = (uint8_t)tscProcessDescribeTable(pSql); + pRes->code = (uint8_t)tscProcessDescribeTable(pSql); } else if (pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { /* * set the qhandle to be 1 in order to pass the qhandle check, and to call partial release function to * free allocated resources and remove the SqlObj from sql query linked list */ - pSql->res.qhandle = 0x1; - pSql->res.numOfRows = 0; + pRes->qhandle = 0x1; + pRes->numOfRows = 0; } else if (pCmd->command == TSDB_SQL_RESET_CACHE) { taosCacheEmpty(tscCacheHandle); + pRes->code = TSDB_CODE_SUCCESS; } else if (pCmd->command == TSDB_SQL_SERV_VERSION) { - tscProcessServerVer(pSql); + pRes->code = tscProcessServerVer(pSql); } else if (pCmd->command == TSDB_SQL_CLI_VERSION) { - tscProcessClientVer(pSql); + pRes->code = tscProcessClientVer(pSql); } else if (pCmd->command == TSDB_SQL_CURRENT_USER) { - tscProcessCurrentUser(pSql); + pRes->code = tscProcessCurrentUser(pSql); } else if (pCmd->command == TSDB_SQL_CURRENT_DB) { - tscProcessCurrentDB(pSql); + pRes->code = tscProcessCurrentDB(pSql); } else if (pCmd->command == TSDB_SQL_SERV_STATUS) { - tscProcessServStatus(pSql); + pRes->code = tscProcessServStatus(pSql); } else { - pSql->res.code = TSDB_CODE_TSC_INVALID_SQL; + pRes->code = TSDB_CODE_TSC_INVALID_SQL; tscError("%p not support command:%d", pSql, pCmd->command); } // keep the code in local variable in order to avoid invalid read in case of async query - int32_t code = pSql->res.code; + int32_t code = pRes->code; if (code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, code); } else { diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 563c9fa84e..39a757795e 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -67,8 +67,7 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc SQLFunctionCtx *pCtx = &pReducer->pCtx[i]; SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, i); - pCtx->aOutputBuf = - pReducer->pResultBuf->data + pExpr->offset * pReducer->resColModel->capacity; + pCtx->aOutputBuf = pReducer->pResultBuf->data + pExpr->offset * pReducer->resColModel->capacity; pCtx->order = pQueryInfo->order.order; pCtx->functionId = pExpr->functionId; @@ -160,7 +159,6 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd if (pMemBuffer == NULL) { tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, numOfBuffer); - tscError("%p pMemBuffer is NULL", pMemBuffer); pRes->code = TSDB_CODE_TSC_APP_ERROR; return; @@ -168,7 +166,6 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd if (pDesc->pColumnModel == NULL) { tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, numOfBuffer); - tscError("%p no local buffer or intermediate result format model", pSql); pRes->code = TSDB_CODE_TSC_APP_ERROR; return; @@ -188,7 +185,6 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd if (numOfFlush == 0 || numOfBuffer == 0) { tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, numOfBuffer); tscDebug("%p retrieved no data", pSql); - return; } @@ -279,6 +275,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd taosTFree(pReducer); return; } + param->pLocalData = pReducer->pLocalDataSrc; param->pDesc = pReducer->pDesc; param->num = pReducer->pLocalDataSrc[0]->pMemBuffer->numOfElemsPerPage; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 47bfe0fcdc..f214e91f45 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -180,7 +180,7 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload, } else if (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0) { *(uint8_t *)payload = TSDB_DATA_BOOL_NULL; } else { - return tscInvalidSQLErrMsg(msg, "invalid bool data", pToken->z); + return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z); } } else if (pToken->type == TK_INTEGER) { iv = strtoll(pToken->z, NULL, 10); @@ -439,8 +439,8 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[ int16_t type = sToken.type; if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL && type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || (sToken.n == 0) || (type == TK_RP)) { - tscInvalidSQLErrMsg(error, "invalid data or symbol", sToken.z); - *code = TSDB_CODE_TSC_INVALID_SQL; + tscSQLSyntaxErrMsg(error, "invalid data or symbol", sToken.z); + *code = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; return -1; } @@ -472,7 +472,7 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[ bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); int32_t ret = tsParseOneColumnData(pSchema, &sToken, start, error, str, isPrimaryKey, timePrec); if (ret != TSDB_CODE_SUCCESS) { - *code = TSDB_CODE_TSC_INVALID_SQL; + *code = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; return -1; // NOTE: here 0 mean error! } @@ -568,8 +568,8 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMe sToken = tStrGetToken(*str, &index, false, 0, NULL); *str += index; if (sToken.n == 0 || sToken.type != TK_RP) { - tscInvalidSQLErrMsg(error, ") expected", *str); - *code = TSDB_CODE_TSC_INVALID_SQL; + tscSQLSyntaxErrMsg(error, ") expected", *str); + *code = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; return -1; } @@ -578,7 +578,7 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMe if (numOfRows <= 0) { strcpy(error, "no any data points"); - *code = TSDB_CODE_TSC_INVALID_SQL; + *code = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; return -1; } else { return numOfRows; @@ -943,7 +943,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { sToken = tStrGetToken(sql, &index, false, 0, NULL); sql += index; if (sToken.n == 0 || sToken.type != TK_RP) { - return tscInvalidSQLErrMsg(pCmd->payload, ") expected", sToken.z); + return tscSQLSyntaxErrMsg(pCmd->payload, ") expected", sToken.z); } pCmd->payloadLen = sizeof(pTag->name) + sizeof(pTag->dataLen) + pTag->dataLen; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 9c677dc555..de608961c2 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -33,6 +33,8 @@ #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" +#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) + // -1 is tbname column index, so here use the -3 as the initial value #define COLUMN_INDEX_INITIAL_VAL (-3) #define COLUMN_INDEX_INITIALIZER \ @@ -45,6 +47,10 @@ typedef struct SColumnList { // todo refactor SColumnIndex ids[TSDB_MAX_COLUMNS]; } SColumnList; +typedef struct SConvertFunc { + int32_t originFuncId; + int32_t execFuncId; +} SConvertFunc; static SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t colIndex, int32_t tableIndex); static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); @@ -184,7 +190,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { int32_t code = TSDB_CODE_SUCCESS; if (!pInfo->valid) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), pInfo->pzErrMsg); + return tscSQLSyntaxErrMsg(tscGetErrorMsgPayload(pCmd), NULL, pInfo->pzErrMsg); } SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); @@ -1507,13 +1513,13 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t return TSDB_CODE_SUCCESS; } -static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, int32_t functionID, char* aliasName, +static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc, char* aliasName, int32_t resColIdx, SColumnIndex* pColIndex) { int16_t type = 0; int16_t bytes = 0; - char columnName[TSDB_COL_NAME_LEN] = {0}; const char* msg1 = "not support column types"; + int32_t functionID = cvtFunc.execFuncId; if (functionID == TSDB_FUNC_SPREAD) { if (pSchema[pColIndex->columnIndex].type == TSDB_DATA_TYPE_BINARY || @@ -1529,16 +1535,21 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS type = pSchema[pColIndex->columnIndex].type; bytes = pSchema[pColIndex->columnIndex].bytes; } - + if (aliasName != NULL) { tstrncpy(columnName, aliasName, sizeof(columnName)); } else { - getRevisedName(columnName, functionID, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name); + getRevisedName(columnName, cvtFunc.originFuncId, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name); } + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false); tstrncpy(pExpr->aliasName, columnName, sizeof(pExpr->aliasName)); + if (cvtFunc.originFuncId == TSDB_FUNC_LAST_ROW && cvtFunc.originFuncId != functionID) { + pExpr->colInfo.flag |= TSDB_COL_NULL; + } + // set reverse order scan data blocks for last query if (functionID == TSDB_FUNC_LAST) { pExpr->numOfParams = 1; @@ -1772,7 +1783,10 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); } - + SConvertFunc cvtFunc = {.originFuncId = functionID, .execFuncId = functionID}; + if (functionID == TSDB_FUNC_LAST_ROW && TSWINDOW_IS_EQUAL(pQueryInfo->window,TSWINDOW_INITIALIZER)) { + cvtFunc.execFuncId = TSDB_FUNC_LAST; + } if (!requireAllFields) { if (pItem->pNode->pParam->nExpr < 1) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -1804,7 +1818,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) { index.columnIndex = j; - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex++, &index) != 0) { + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex++, &index) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } } @@ -1821,8 +1835,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if ((index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) || (index.columnIndex < 0)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } - - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex + i, &index) != 0) { + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex + i, &index) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -1859,7 +1872,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { SColumnIndex index = {.tableIndex = j, .columnIndex = i}; - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex, &index) != 0) { + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex, &index) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -5246,7 +5259,7 @@ static bool tagColumnInGroupby(SSqlGroupbyExpr* pGroupbyExpr, int16_t columnId) for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, j); - if (columnId == pColIndex->colId && pColIndex->flag == TSDB_COL_TAG) { + if (columnId == pColIndex->colId && TSDB_COL_IS_TAG(pColIndex->flag )) { return true; } } @@ -5545,7 +5558,6 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { return checkUpdateTagPrjFunctions(pQueryInfo, pCmd); } } - int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { const char* msg1 = "only one expression allowed"; const char* msg2 = "invalid expression in select clause"; @@ -5815,22 +5827,34 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { int32_t ret = TSDB_CODE_SUCCESS; for (int32_t i = 0; i < pList->nExpr; ++i) { - SSchema* pSchema = pTagSchema + i; + SSchema* pSchema = &pTagSchema[i]; + + char tagVal[TSDB_MAX_TAGS_LEN]; if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { - // validate the length of binary - if (pList->a[i].pVar.nLen + VARSTR_HEADER_SIZE > pSchema->bytes) { + if (pList->a[i].pVar.nLen > pSchema->bytes) { tdDestroyKVRowBuilder(&kvRowBuilder); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } - - char tagVal[TSDB_MAX_TAGS_LEN]; + ret = tVariantDump(&(pList->a[i].pVar), tagVal, pSchema->type, true); + + // check again after the convert since it may be converted from binary to nchar. + if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { + int16_t len = varDataTLen(tagVal); + if (len > pSchema->bytes) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + } + if (ret != TSDB_CODE_SUCCESS) { tdDestroyKVRowBuilder(&kvRowBuilder); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } + + tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); } @@ -6086,6 +6110,10 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } int32_t joinQuery = (pQuerySql->from != NULL && pQuerySql->from->nExpr > 2); + + if (pQuerySql->pWhere) { + pQueryInfo->window = TSWINDOW_INITIALIZER; + } if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index b36767dbb4..1f042b59d6 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -226,17 +226,13 @@ int tscSendMsgToServer(SSqlObj *pSql) { .handle = &pSql->pRpcCtx, .code = 0 }; + // NOTE: the rpc context should be acquired before sending data to server. // Otherwise, the pSql object may have been released already during the response function, which is // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely // cause crash. - if (pObj != NULL && pObj->signature == pObj) { - rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); - return TSDB_CODE_SUCCESS; - } else { - //pObj->signature has been reset by other thread, ignore concurrency problem - return TSDB_CODE_TSC_CONN_KILLED; - } + rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); + return TSDB_CODE_SUCCESS; } void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { @@ -1496,8 +1492,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { char *tmpData = NULL; uint32_t len = pSql->cmd.payloadLen; if (len > 0) { - tmpData = calloc(1, len); - if (NULL == tmpData) { + if ((tmpData = calloc(1, len)) == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1542,8 +1537,7 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // copy payload content to temp buff char *tmpData = 0; if (pCmd->payloadLen > 0) { - tmpData = calloc(1, pCmd->payloadLen + 1); - if (NULL == tmpData) return -1; + if ((tmpData = calloc(1, pCmd->payloadLen + 1)) == NULL) return -1; memcpy(tmpData, pCmd->payload, pCmd->payloadLen); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index f63923e046..1af53d3645 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -597,11 +597,12 @@ int taos_errno(TAOS_RES *tres) { } /* - * In case of invalid sql error, additional information is attached to explain + * In case of invalid sql/sql syntax error, additional information is attached to explain * why the sql is invalid */ static bool hasAdditionalErrorInfo(int32_t code, SSqlCmd *pCmd) { - if (code != TSDB_CODE_TSC_INVALID_SQL) { + if (code != TSDB_CODE_TSC_INVALID_SQL + && code != TSDB_CODE_TSC_SQL_SYNTAX_ERROR) { return false; } @@ -609,9 +610,11 @@ static bool hasAdditionalErrorInfo(int32_t code, SSqlCmd *pCmd) { char *z = NULL; if (len > 0) { - z = strstr(pCmd->payload, "invalid SQL"); + z = strstr(pCmd->payload, "invalid SQL"); + if (z == NULL) { + z = strstr(pCmd->payload, "syntax error"); + } } - return z != NULL; } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index ea979bfae3..79e0011093 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -325,7 +325,7 @@ static int64_t getLaunchTimeDelay(const SSqlStream* pStream) { int64_t delayDelta = maxDelay; if (pStream->intervalTimeUnit != 'n' && pStream->intervalTimeUnit != 'y') { - delayDelta = pStream->slidingTime * tsStreamComputDelayRatio; + delayDelta = (int64_t)(pStream->slidingTime * tsStreamComputDelayRatio); if (delayDelta > maxDelay) { delayDelta = maxDelay; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 7a626bfe5c..e264fa9b33 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -574,8 +574,9 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar SSchema* pColSchema = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); - *s1 = taosArrayInit(p1->num, p1->tagSize); - *s2 = taosArrayInit(p2->num, p2->tagSize); + // int16_t for padding + *s1 = taosArrayInit(p1->num, p1->tagSize - sizeof(int16_t)); + *s2 = taosArrayInit(p2->num, p2->tagSize - sizeof(int16_t)); if (!(checkForDuplicateTagVal(pQueryInfo, p1, pParentSql) && checkForDuplicateTagVal(pQueryInfo, p2, pParentSql))) { return TSDB_CODE_QRY_DUP_JOIN_KEY; @@ -1043,6 +1044,10 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { int32_t numOfExprs = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); pRes->pColumnIndex = calloc(1, sizeof(SColumnIndex) * numOfExprs); + if (pRes->pColumnIndex == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return; + } for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); @@ -1157,7 +1162,8 @@ static void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code); static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); -int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter *pSupporter) { +// TODO +int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter *pSupporter) { SSqlCmd * pCmd = &pSql->cmd; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -1203,7 +1209,9 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter // this data needs to be transfer to support struct memset(&pNewQueryInfo->fieldsInfo, 0, sizeof(SFieldInfo)); - tscTagCondCopy(&pSupporter->tagCond, &pNewQueryInfo->tagCond);//pNewQueryInfo->tagCond; + if (tscTagCondCopy(&pSupporter->tagCond, &pNewQueryInfo->tagCond) != 0) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } pNew->cmd.numOfCols = 0; pNewQueryInfo->intervalTime = 0; @@ -1300,52 +1308,75 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter pNewQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; } - return tscProcessSql(pNew); + return TSDB_CODE_SUCCESS; } -int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { +void tscHandleMasterJoinQuery(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); assert((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0); + int32_t code = TSDB_CODE_SUCCESS; + // todo add test SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); if (pState == NULL) { - pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pSql->res.code; + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; } pState->numOfTotal = pQueryInfo->numOfTables; pState->numOfRemain = pState->numOfTotal; + bool hasEmptySub = false; + tscDebug("%p start subquery, total:%d", pSql, pQueryInfo->numOfTables); for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { SJoinSupporter *pSupporter = tscCreateJoinSupporter(pSql, pState, i); if (pSupporter == NULL) { // failed to create support struct, abort current query tscError("%p tableIndex:%d, failed to allocate join support object, abort further query", pSql, i); - pState->numOfRemain = i; - pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - if (0 == i) { - taosTFree(pState); - } - return pSql->res.code; + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; } - int32_t code = tscLaunchJoinSubquery(pSql, i, pSupporter); + code = tscCreateJoinSubquery(pSql, i, pSupporter); if (code != TSDB_CODE_SUCCESS) { // failed to create subquery object, quit query tscDestroyJoinSupporter(pSupporter); - pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - if (0 == i) { - taosTFree(pState); - } + goto _error; + } + + SSqlObj* pSub = pSql->pSubs[i]; + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSub->cmd, 0, 0); + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) && (pTableMetaInfo->vgroupList->numOfVgroups == 0)) { + hasEmptySub = true; break; } } - pSql->cmd.command = (pSql->numOfSubs <= 0)? TSDB_SQL_RETRIEVE_EMPTY_RESULT:TSDB_SQL_TABLE_JOIN_RETRIEVE; - - return TSDB_CODE_SUCCESS; + if (hasEmptySub) { // at least one subquery is empty, do nothing and return + freeJoinSubqueryObj(pSql); + pSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + (*pSql->fp)(pSql->param, pSql, 0); + } else { + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + SSqlObj* pSub = pSql->pSubs[i]; + if ((code = tscProcessSql(pSub)) != TSDB_CODE_SUCCESS) { + pState->numOfRemain = i - 1; // the already sent reques will continue and do not go to the error process routine + break; + } + } + + pSql->cmd.command = TSDB_SQL_TABLE_JOIN_RETRIEVE; + } + + return; + + _error: + pRes->code = code; + tscQueueAsyncRes(pSql); } static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs, SSubqueryState* pState) { @@ -1384,7 +1415,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { const uint32_t nBufferSize = (1u << 16); // 64KB - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); pSql->numOfSubs = pTableMetaInfo->vgroupList->numOfVgroups; @@ -1399,9 +1430,20 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { } pSql->pSubs = calloc(pSql->numOfSubs, POINTER_BYTES); - + tscDebug("%p retrieved query data from %d vnode(s)", pSql, pSql->numOfSubs); SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); + + if (pSql->pSubs == NULL || pState == NULL) { + taosTFree(pState); + taosTFree(pSql->pSubs); + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pSql->numOfSubs); + + tscQueueAsyncRes(pSql); + return ret; + } + pState->numOfTotal = pSql->numOfSubs; pState->numOfRemain = pSql->numOfSubs; @@ -2033,8 +2075,21 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) { numOfRes = (int32_t)(MIN(numOfRes, pSql->pSubs[i]->res.numOfRows)); } + if (numOfRes == 0) { + return; + } + int32_t totalSize = tscGetResRowLength(pQueryInfo->exprList); - pRes->pRsp = realloc(pRes->pRsp, numOfRes * totalSize); + + assert(numOfRes * totalSize > 0); + char* tmp = realloc(pRes->pRsp, numOfRes * totalSize); + if (tmp == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return; + } else { + pRes->pRsp = tmp; + } + pRes->data = pRes->pRsp; char* data = pRes->data; @@ -2073,6 +2128,12 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { pRes->buffer = calloc(numOfExprs, POINTER_BYTES); pRes->length = calloc(numOfExprs, sizeof(int32_t)); + if (pRes->tsrow == NULL || pRes->buffer == NULL || pRes->length == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + tscQueueAsyncRes(pSql); + return; + } + tscRestoreSQLFuncForSTableQuery(pQueryInfo); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 49f7c91397..e1a0ff69f9 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -254,15 +254,12 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { pRes->numOfCols = numOfOutput; pRes->tsrow = calloc(numOfOutput, POINTER_BYTES); - pRes->length = calloc(numOfOutput, sizeof(int32_t)); // todo refactor + pRes->length = calloc(numOfOutput, sizeof(int32_t)); pRes->buffer = calloc(numOfOutput, POINTER_BYTES); // not enough memory if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) { taosTFree(pRes->tsrow); - taosTFree(pRes->buffer); - taosTFree(pRes->length); - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; return pRes->code; } @@ -281,13 +278,14 @@ void tscDestroyResPointerInfo(SSqlRes* pRes) { } taosTFree(pRes->pRsp); + taosTFree(pRes->tsrow); taosTFree(pRes->length); - + taosTFree(pRes->buffer); + taosTFree(pRes->pGroupRec); taosTFree(pRes->pColumnIndex); - taosTFree(pRes->buffer); - + if (pRes->pArithSup != NULL) { taosTFree(pRes->pArithSup->data); taosTFree(pRes->pArithSup); @@ -1052,7 +1050,7 @@ void tscSqlExprInfoDestroy(SArray* pExprInfo) { taosArrayDestroy(pExprInfo); } -void tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) { +int32_t tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) { assert(src != NULL && dst != NULL); size_t size = taosArrayGetSize(src); @@ -1064,7 +1062,7 @@ void tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) if (deepcopy) { SSqlExpr* p1 = calloc(1, sizeof(SSqlExpr)); if (p1 == NULL) { - assert(0); + return -1; } *p1 = *pExpr; @@ -1078,6 +1076,8 @@ void tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) } } } + + return 0; } SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { @@ -1324,11 +1324,14 @@ bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t return false; } -void tscTagCondCopy(STagCond* dest, const STagCond* src) { +int32_t tscTagCondCopy(STagCond* dest, const STagCond* src) { memset(dest, 0, sizeof(STagCond)); if (src->tbnameCond.cond != NULL) { dest->tbnameCond.cond = strdup(src->tbnameCond.cond); + if (dest->tbnameCond.cond == NULL) { + return -1; + } } dest->tbnameCond.uid = src->tbnameCond.uid; @@ -1337,7 +1340,7 @@ void tscTagCondCopy(STagCond* dest, const STagCond* src) { dest->relType = src->relType; if (src->pCond == NULL) { - return; + return 0; } size_t s = taosArrayGetSize(src->pCond); @@ -1354,7 +1357,7 @@ void tscTagCondCopy(STagCond* dest, const STagCond* src) { assert(pCond->cond != NULL); c.cond = malloc(c.len); if (c.cond == NULL) { - assert(0); + return -1; } memcpy(c.cond, pCond->cond, c.len); @@ -1362,6 +1365,8 @@ void tscTagCondCopy(STagCond* dest, const STagCond* src) { taosArrayPush(dest->pCond, &c); } + + return 0; } void tscTagCondRelease(STagCond* pTagCond) { @@ -1855,7 +1860,10 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } } - tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond); + if (tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond) != 0) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } if (pQueryInfo->fillType != TSDB_FILL_NONE) { pNewQueryInfo->fillVal = malloc(pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); @@ -1884,7 +1892,10 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; - tscSqlExprCopy(pNewQueryInfo->exprList, pQueryInfo->exprList, uid, true); + if (tscSqlExprCopy(pNewQueryInfo->exprList, pQueryInfo->exprList, uid, true) != 0) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } doSetSqlExprAndResultFieldInfo(pQueryInfo, pNewQueryInfo, uid); @@ -2029,10 +2040,37 @@ bool tscIsUpdateQuery(SSqlObj* pSql) { return ((pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) || TSDB_SQL_USE_DB == pCmd->command); } +int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* sql) { + const char* msgFormat1 = "syntax error near \'%s\'"; + const char* msgFormat2 = "syntax error near \'%s\' (%s)"; + const char* msgFormat3 = "%s"; + + const char* prefix = "syntax error"; + const int32_t BACKWARD_CHAR_STEP = 0; + + if (sql == NULL) { + assert(additionalInfo != NULL); + sprintf(msg, msgFormat1, additionalInfo); + return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + } + + char buf[64] = {0}; // only extract part of sql string + strncpy(buf, (sql - BACKWARD_CHAR_STEP), tListLen(buf) - 1); + + if (additionalInfo != NULL) { + sprintf(msg, msgFormat2, buf, additionalInfo); + } else { + const char* msgFormat = (0 == strncmp(sql, prefix, strlen(prefix))) ? msgFormat3 : msgFormat1; + sprintf(msg, msgFormat, buf); + } + + return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + +} int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* sql) { const char* msgFormat1 = "invalid SQL: %s"; - const char* msgFormat2 = "invalid SQL: syntax error near \"%s\" (%s)"; - const char* msgFormat3 = "invalid SQL: syntax error near \"%s\""; + const char* msgFormat2 = "invalid SQL: \'%s\' (%s)"; + const char* msgFormat3 = "invalid SQL: \'%s\'"; const int32_t BACKWARD_CHAR_STEP = 0; @@ -2258,4 +2296,4 @@ bool tscSetSqlOwner(SSqlObj* pSql) { void tscClearSqlOwner(SSqlObj* pSql) { assert(taosCheckPthreadValid(pSql->owner)); atomic_store_64(&pSql->owner, 0); -} \ No newline at end of file +} diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 248c996999..49c9e6b726 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -114,7 +114,7 @@ int64_t taosAddNatualInterval(int64_t key, int64_t intervalTime, char timeUnit, intervalTime *= 12; } - int mon = tm.tm_year * 12 + tm.tm_mon + intervalTime; + int mon = (int)(tm.tm_year * 12 + tm.tm_mon + intervalTime); tm.tm_year = mon / 12; tm.tm_mon = mon % 12; @@ -176,10 +176,10 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in if (timeUnit == 'y') { tm.tm_mon = 0; - tm.tm_year = tm.tm_year / slidingTime * slidingTime; + tm.tm_year = (int)(tm.tm_year / slidingTime * slidingTime); } else { int mon = tm.tm_year * 12 + tm.tm_mon; - mon = mon / slidingTime * slidingTime; + mon = (int)(mon / slidingTime * slidingTime); tm.tm_year = mon / 12; tm.tm_mon = mon % 12; } @@ -189,7 +189,10 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in start *= 1000L; } } else { - start = ((start - intervalTime) / slidingTime + 1) * slidingTime; + int64_t delta = startTime - intervalTime; + int32_t factor = delta > 0? 1:-1; + + start = (delta / slidingTime + factor) * slidingTime; if (timeUnit == 'd' || timeUnit == 'w') { /* diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 951c511022..d8e5c8f1d7 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -98,6 +98,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS, 0, 0x0212, "Action in TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED, 0, 0x0213, "Disconnected from service") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH, 0, 0x0214, "No write permission") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED, 0, 0x0215, "Connection killed") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR, 0, 0x0216, "Syntax errr in SQL") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, 0, 0x0300, "Message not processed") diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 0fe63a740e..e2df886320 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -170,6 +170,13 @@ enum _mgmt_table { #define TSDB_COL_NORMAL 0x0u // the normal column of the table #define TSDB_COL_TAG 0x1u // the tag column type #define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column +#define TSDB_COL_NULL 0x4u // the column filter NULL or not + +#define TSDB_COL_IS_TAG(f) (((f&(~(TSDB_COL_NULL)))&TSDB_COL_TAG) != 0) +#define TSDB_COL_IS_NORMAL_COL(f) ((f&(~(TSDB_COL_NULL))) == TSDB_COL_NORMAL) +#define TSDB_COL_IS_UD_COL(f) ((f&(~(TSDB_COL_NULL))) == TSDB_COL_UDC) +#define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) + extern char *taosMsg[]; diff --git a/src/kit/CMakeLists.txt b/src/kit/CMakeLists.txt index 3bacd426f3..77db79e220 100644 --- a/src/kit/CMakeLists.txt +++ b/src/kit/CMakeLists.txt @@ -3,5 +3,3 @@ PROJECT(TDengine) ADD_SUBDIRECTORY(shell) ADD_SUBDIRECTORY(taosdemo) -#ADD_SUBDIRECTORY(taosClusterTest) -ADD_SUBDIRECTORY(taosnetwork) diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt index 0305d9f1cc..c86cac281c 100644 --- a/src/kit/shell/CMakeLists.txt +++ b/src/kit/shell/CMakeLists.txt @@ -24,7 +24,12 @@ ELSEIF (TD_WINDOWS) LIST(APPEND SRC ./src/shellWindows.c) ADD_EXECUTABLE(shell ${SRC}) TARGET_LINK_LIBRARIES(shell taos_static) - SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) + + IF (TD_POWER) + SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME power) + ELSE () + SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) + ENDIF () ELSEIF (TD_DARWIN) LIST(APPEND SRC ./src/shellEngine.c) LIST(APPEND SRC ./src/shellMain.c) diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h index d47f87965b..dd62df170a 100644 --- a/src/kit/shell/inc/shell.h +++ b/src/kit/shell/inc/shell.h @@ -50,6 +50,9 @@ typedef struct SShellArguments { char* commands; int abort; int port; + int endPort; + int pktLen; + char* netTestRole; } SShellArguments; /**************** Function declarations ****************/ diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index c74eeb7f59..6c09d5c9d0 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -46,6 +46,9 @@ static struct argp_option options[] = { {"thread", 'T', "THREADNUM", 0, "Number of threads when using multi-thread to import data."}, {"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."}, {"timezone", 't', "TIMEZONE", 0, "Time zone of the shell, default is local."}, + {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is NULL, valid option: client | server."}, + {"endport", 'e', "ENDPORT", 0, "Net test end port, default is 6042."}, + {"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."}, {0}}; static error_t parse_opt(int key, char *arg, struct argp_state *state) { @@ -65,6 +68,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'P': if (arg) { tsDnodeShellPort = atoi(arg); + arguments->port = atoi(arg); } else { fprintf(stderr, "Invalid port\n"); return -1; @@ -126,6 +130,29 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'd': arguments->database = arg; break; + + case 'n': + arguments->netTestRole = arg; + break; + + case 'e': + if (arg) { + arguments->endPort = atoi(arg); + } else { + fprintf(stderr, "Invalid end port\n"); + return -1; + } + break; + + case 'l': + if (arg) { + arguments->pktLen = atoi(arg); + } else { + fprintf(stderr, "Invalid packet length\n"); + return -1; + } + break; + case OPT_ABORT: arguments->abort = 1; break; diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 44de6641f6..6cb7c669cc 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -15,6 +15,7 @@ #include "os.h" #include "shell.h" +#include "tnettest.h" pthread_t pid; @@ -60,7 +61,10 @@ SShellArguments args = { .file = "\0", .dir = "\0", .threadNum = 5, - .commands = NULL + .commands = NULL, + .endPort = 6042, + .pktLen = 1000, + .netTestRole = NULL }; /* @@ -75,6 +79,11 @@ int main(int argc, char* argv[]) { shellParseArgument(argc, argv, &args); + if (args.netTestRole && args.netTestRole[0] != 0) { + taosNetTest(args.host, (uint16_t)args.port, (uint16_t)args.endPort, args.pktLen, args.netTestRole); + exit(0); + } + /* Initialize the shell */ TAOS* con = shellInit(&args); if (con == NULL) { diff --git a/src/kit/taosnetwork/CMakeLists.txt b/src/kit/taosnetwork/CMakeLists.txt deleted file mode 100644 index 9d2a5ba3f8..0000000000 --- a/src/kit/taosnetwork/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) -PROJECT(TDengine) - -IF (TD_LINUX) - AUX_SOURCE_DIRECTORY(. SRC) - ADD_EXECUTABLE(taosClient client.c) - ADD_EXECUTABLE(taosServer server.c) - TARGET_LINK_LIBRARIES( taosServer -lpthread -lm -lrt ) - TARGET_LINK_LIBRARIES( taosClient -lpthread -lm -lrt ) -ENDIF () diff --git a/src/kit/taosnetwork/client.c b/src/kit/taosnetwork/client.c deleted file mode 100644 index b7db2ba0a2..0000000000 --- a/src/kit/taosnetwork/client.c +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define MAX_PKG_LEN (64*1000) -#define BUFFER_SIZE (MAX_PKG_LEN + 1024) -#define TEST_FQDN_LEN 128 -#define TEST_IPv4ADDR_LEN 16 - -typedef struct { - uint16_t port; - uint32_t hostIp; - char fqdn[TEST_FQDN_LEN]; - uint16_t pktLen; -} info_s; - -typedef struct Arguments { - char host[TEST_IPv4ADDR_LEN]; - char fqdn[TEST_FQDN_LEN]; - uint16_t port; - uint16_t max_port; - uint16_t pktLen; -} SArguments; - -static struct argp_option options[] = { - {0, 'h', "host ip", 0, "The host ip to connect to TDEngine. Default is localhost.", 0}, - {0, 'p', "port", 0, "The TCP or UDP port number to use for the connection. Default is 6030.", 1}, - {0, 'm', "max port", 0, "The max TCP or UDP port number to use for the connection. Default is 6042.", 2}, - {0, 'f', "host fqdn", 0, "The host fqdn to connect to TDEngine.", 3}, - {0, 'l', "test pkg len", 0, "The len of pkg for test. Default is 1000 Bytes, max not greater than 64k Bytes.\nNotes: This parameter must be consistent between the client and the server.", 3}}; - -static error_t parse_opt(int key, char *arg, struct argp_state *state) { - wordexp_t full_path; - SArguments *arguments = state->input; - switch (key) { - case 'h': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid host ip %s\n", arg); - return -1; - } - strcpy(arguments->host, full_path.we_wordv[0]); - wordfree(&full_path); - break; - case 'p': - arguments->port = atoi(arg); - break; - case 'm': - arguments->max_port = atoi(arg); - break; - case 'l': - arguments->pktLen = atoi(arg); - break; - case 'f': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid host fqdn %s\n", arg); - return -1; - } - strcpy(arguments->fqdn, full_path.we_wordv[0]); - wordfree(&full_path); - break; - - default: - return ARGP_ERR_UNKNOWN; - } - return 0; -} - -static struct argp argp = {options, parse_opt, 0, 0}; - -int checkTcpPort(info_s *info) { - int clientSocket; - - struct sockaddr_in serverAddr; - char sendbuf[BUFFER_SIZE]; - char recvbuf[BUFFER_SIZE]; - int iDataNum = 0; - if ((clientSocket = socket(AF_INET, SOCK_STREAM, 0)) < 0) { - printf("socket() fail: %s\n", strerror(errno)); - return -1; - } - - // set send and recv overtime - struct timeval timeout; - timeout.tv_sec = 2; //s - timeout.tv_usec = 0; //us - if (setsockopt(clientSocket, SOL_SOCKET,SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { - perror("setsockopt send timer failed:"); - } - if (setsockopt(clientSocket, SOL_SOCKET,SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { - perror("setsockopt recv timer failed:"); - } - - serverAddr.sin_family = AF_INET; - serverAddr.sin_port = htons(info->port); - - serverAddr.sin_addr.s_addr = info->hostIp; - - //printf("=================================\n"); - if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { - printf("connect() fail: %s\t", strerror(errno)); - return -1; - } - //printf("Connect to: %s:%d...success\n", host, port); - memset(sendbuf, 0, BUFFER_SIZE); - memset(recvbuf, 0, BUFFER_SIZE); - - struct in_addr ipStr; - memcpy(&ipStr, &info->hostIp, 4); - sprintf(sendbuf, "client send tcp pkg to %s:%d, content: 1122334455", inet_ntoa(ipStr), info->port); - sprintf(sendbuf + info->pktLen - 16, "1122334455667788"); - - send(clientSocket, sendbuf, info->pktLen, 0); - - memset(recvbuf, 0, BUFFER_SIZE); - int nleft, nread; - char *ptr = recvbuf; - nleft = info->pktLen; - while (nleft > 0) { - nread = recv(clientSocket, ptr, BUFFER_SIZE, 0);; - - if (nread == 0) { - break; - } else if (nread < 0) { - if (errno == EINTR) { - continue; - } else { - printf("recv ack pkg from TCP port: %d fail:%s.\n", info->port, strerror(errno)); - close(clientSocket); - return -1; - } - } else { - nleft -= nread; - ptr += nread; - iDataNum += nread; - } - } - - if (iDataNum < info->pktLen) { - printf("recv ack pkg len: %d, less than req pkg len: %d from tcp port: %d\n", iDataNum, info->pktLen, info->port); - return -1; - } - //printf("Read ack pkg len:%d from tcp port: %d, buffer: %s %s\n", info->pktLen, port, recvbuf, recvbuf+iDataNum-8); - - close(clientSocket); - return 0; -} - -int checkUdpPort(info_s *info) { - int clientSocket; - - struct sockaddr_in serverAddr; - char sendbuf[BUFFER_SIZE]; - char recvbuf[BUFFER_SIZE]; - int iDataNum = 0; - if ((clientSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { - perror("socket"); - return -1; - } - - // set overtime - struct timeval timeout; - timeout.tv_sec = 2; //s - timeout.tv_usec = 0; //us - if (setsockopt(clientSocket, SOL_SOCKET,SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { - perror("setsockopt send timer failed:"); - } - if (setsockopt(clientSocket, SOL_SOCKET,SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { - perror("setsockopt recv timer failed:"); - } - - serverAddr.sin_family = AF_INET; - serverAddr.sin_port = htons(info->port); - serverAddr.sin_addr.s_addr = info->hostIp; - - memset(sendbuf, 0, BUFFER_SIZE); - memset(recvbuf, 0, BUFFER_SIZE); - - struct in_addr ipStr; - memcpy(&ipStr, &info->hostIp, 4); - sprintf(sendbuf, "client send udp pkg to %s:%d, content: 1122334455", inet_ntoa(ipStr), info->port); - sprintf(sendbuf + info->pktLen - 16, "1122334455667788"); - - socklen_t sin_size = sizeof(*(struct sockaddr *)&serverAddr); - - int code = sendto(clientSocket, sendbuf, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int)sin_size); - if (code < 0) { - perror("sendto"); - return -1; - } - - iDataNum = recvfrom(clientSocket, recvbuf, BUFFER_SIZE, 0, (struct sockaddr *)&serverAddr, &sin_size); - - if (iDataNum < info->pktLen) { - printf("Read ack pkg len: %d, less than req pkg len: %d from udp port: %d\t\t", iDataNum, info->pktLen, info->port); - return -1; - } - - //printf("Read ack pkg len:%d from udp port: %d, buffer: %s %s\n", info->pktLen, port, recvbuf, recvbuf+iDataNum-8); - close(clientSocket); - return 0; -} - -int32_t getIpFromFqdn(const char *fqdn, uint32_t* ip) { - struct addrinfo hints = {0}; - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - - struct addrinfo *result = NULL; - - int32_t ret = getaddrinfo(fqdn, NULL, &hints, &result); - if (result) { - struct sockaddr *sa = result->ai_addr; - struct sockaddr_in *si = (struct sockaddr_in*)sa; - struct in_addr ia = si->sin_addr; - *ip = ia.s_addr; - freeaddrinfo(result); - return 0; - } else { - printf("Failed get the ip address from fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret)); - return -1; - } -} - -void checkPort(uint32_t hostIp, uint16_t startPort, uint16_t maxPort, uint16_t pktLen) { - int ret; - info_s info; - memset(&info, 0, sizeof(info_s)); - info.hostIp = hostIp; - info.pktLen = pktLen; - - for (uint16_t port = startPort; port <= maxPort; port++) { - //printf("test: %s:%d\n", info.host, port); - printf("\n"); - - info.port = port; - ret = checkTcpPort(&info); - if (ret != 0) { - printf("tcp port:%d test fail.\t\n", port); - } else { - printf("tcp port:%d test ok.\t\t", port); - } - - ret = checkUdpPort(&info); - if (ret != 0) { - printf("udp port:%d test fail.\t\n", port); - } else { - printf("udp port:%d test ok.\t\t", port); - } - } - - printf("\n"); - return ; -} - -int main(int argc, char *argv[]) { - SArguments arguments = {"127.0.0.1", "", 6030, 6042, 1000}; - int ret; - - argp_parse(&argp, argc, argv, 0, 0, &arguments); - if (arguments.pktLen > MAX_PKG_LEN) { - printf("test pkg len overflow: %d, max len not greater than %d bytes\n", arguments.pktLen, MAX_PKG_LEN); - exit(0); - } - - printf("host ip: %s\thost fqdn: %s\tport: %d\tmax_port: %d\tpkgLen: %d\n", arguments.host, arguments.fqdn, arguments.port, arguments.max_port, arguments.pktLen); - - if (arguments.host[0] != 0) { - printf("\nstart connect to %s test:\n", arguments.host); - checkPort(inet_addr(arguments.host), arguments.port, arguments.max_port, arguments.pktLen); - printf("\n"); - } - - if (arguments.fqdn[0] != 0) { - uint32_t hostIp = 0; - ret = getIpFromFqdn(arguments.fqdn, &hostIp); - if (ret) { - printf("\n"); - return 0; - } - printf("\nstart connetc to %s test:\n", arguments.fqdn); - checkPort(hostIp, arguments.port, arguments.max_port, arguments.pktLen); - printf("\n"); - } - - return 0; -} diff --git a/src/kit/taosnetwork/server.c b/src/kit/taosnetwork/server.c deleted file mode 100644 index 97be1d3b63..0000000000 --- a/src/kit/taosnetwork/server.c +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define MAX_PKG_LEN (64*1000) -#define BUFFER_SIZE (MAX_PKG_LEN + 1024) - -typedef struct { - int port; - uint16_t pktLen; -} info_s; - -typedef struct Arguments { - char * host; - uint16_t port; - uint16_t max_port; - uint16_t pktLen; -} SArguments; - -static struct argp_option options[] = { - {0, 'h', "host", 0, "The host to connect to TDEngine. Default is localhost.", 0}, - {0, 'p', "port", 0, "The TCP or UDP port number to use for the connection. Default is 6041.", 1}, - {0, 'm', "max port", 0, "The max TCP or UDP port number to use for the connection. Default is 6060.", 2}, - {0, 'l', "test pkg len", 0, "The len of pkg for test. Default is 1000 Bytes, max not greater than 64k Bytes.\nNotes: This parameter must be consistent between the client and the server.", 3}}; - -static error_t parse_opt(int key, char *arg, struct argp_state *state) { - - SArguments *arguments = state->input; - switch (key) { - case 'h': - arguments->host = arg; - break; - case 'p': - arguments->port = atoi(arg); - break; - case 'm': - arguments->max_port = atoi(arg); - break; - case 'l': - arguments->pktLen = atoi(arg); - break; - - default: - return ARGP_ERR_UNKNOWN; - } - return 0; -} - -static struct argp argp = {options, parse_opt, 0, 0}; - -static void *bindTcpPort(void *sarg) { - info_s *pinfo = (info_s *)sarg; - int port = pinfo->port; - int serverSocket; - - struct sockaddr_in server_addr; - struct sockaddr_in clientAddr; - int addr_len = sizeof(clientAddr); - int client; - char buffer[BUFFER_SIZE]; - int iDataNum = 0; - - if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { - printf("socket() fail: %s", strerror(errno)); - return NULL; - } - - bzero(&server_addr, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - server_addr.sin_addr.s_addr = htonl(INADDR_ANY); - - if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { - printf("port:%d bind() fail: %s", port, strerror(errno)); - return NULL; - } - - if (listen(serverSocket, 5) < 0) { - printf("listen() fail: %s", strerror(errno)); - return NULL; - } - - //printf("Bind port: %d success\n", port); - while (1) { - client = accept(serverSocket, (struct sockaddr *)&clientAddr, (socklen_t *)&addr_len); - if (client < 0) { - printf("accept() fail: %s", strerror(errno)); - continue; - } - - memset(buffer, 0, BUFFER_SIZE); - int nleft, nread; - char *ptr = buffer; - nleft = pinfo->pktLen; - while (nleft > 0) { - nread = recv(client, ptr, BUFFER_SIZE, 0); - - if (nread == 0) { - break; - } else if (nread < 0) { - if (errno == EINTR) { - continue; - } else { - printf("recv Client: %s pkg from TCP port: %d fail:%s.\n", inet_ntoa(clientAddr.sin_addr), port, strerror(errno)); - close(serverSocket); - return NULL; - } - } else { - nleft -= nread; - ptr += nread; - iDataNum += nread; - } - } - - printf("recv Client: %s pkg from TCP port: %d, pkg len: %d\n", inet_ntoa(clientAddr.sin_addr), port, iDataNum); - if (iDataNum > 0) { - send(client, buffer, iDataNum, 0); - } - } - - close(serverSocket); - return NULL; -} - -static void *bindUdpPort(void *sarg) { - info_s *pinfo = (info_s *)sarg; - int port = pinfo->port; - int serverSocket; - - struct sockaddr_in server_addr; - struct sockaddr_in clientAddr; - char buffer[BUFFER_SIZE]; - int iDataNum; - - if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { - perror("socket"); - return NULL; - } - - bzero(&server_addr, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - server_addr.sin_addr.s_addr = htonl(INADDR_ANY); - - if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { - perror("connect"); - return NULL; - } - - socklen_t sin_size; - - while (1) { - memset(buffer, 0, BUFFER_SIZE); - - sin_size = sizeof(*(struct sockaddr *)&server_addr); - - iDataNum = recvfrom(serverSocket, buffer, BUFFER_SIZE, 0, (struct sockaddr *)&clientAddr, &sin_size); - - if (iDataNum < 0) { - perror("recvfrom null"); - continue; - } - if (iDataNum > 0) { - printf("recv Client: %s pkg from UDP port: %d, pkg len: %d\n", inet_ntoa(clientAddr.sin_addr), port, iDataNum); - //printf("Read msg from udp:%s ... %s\n", buffer, buffer+iDataNum-16); - - sendto(serverSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int)sin_size); - } - } - - close(serverSocket); - return NULL; -} - - -int main(int argc, char *argv[]) { - SArguments arguments = {"127.0.0.1", 6030, 6042, 1000}; - argp_parse(&argp, argc, argv, 0, 0, &arguments); - if (arguments.pktLen > MAX_PKG_LEN) { - printf("test pkg len overflow: %d, max len not greater than %d bytes\n", arguments.pktLen, MAX_PKG_LEN); - exit(0); - } - - int port = arguments.port; - - int num = arguments.max_port - arguments.port + 1; - - if (num < 0) { - num = 1; - } - pthread_t *pids = malloc(2 * num * sizeof(pthread_t)); - info_s * tinfos = malloc(num * sizeof(info_s)); - info_s * uinfos = malloc(num * sizeof(info_s)); - - for (size_t i = 0; i < num; i++) { - info_s *tcpInfo = tinfos + i; - tcpInfo->port = port + i; - tcpInfo->pktLen = arguments.pktLen; - - if (pthread_create(pids + i, NULL, bindTcpPort, tcpInfo) != 0) - { - printf("create thread fail, port:%d.\n", port); - exit(-1); - } - - info_s *udpInfo = uinfos + i; - udpInfo->port = port + i; - if (pthread_create(pids + num + i, NULL, bindUdpPort, udpInfo) != 0) - { - printf("create thread fail, port:%d.\n", port); - exit(-1); - } - } - - for (int i = 0; i < num; i++) { - pthread_join(pids[i], NULL); - pthread_join(pids[(num + i)], NULL); - } -} diff --git a/src/plugins/http/inc/httpQueue.h b/src/plugins/http/inc/httpQueue.h new file mode 100644 index 0000000000..a4590719ff --- /dev/null +++ b/src/plugins/http/inc/httpQueue.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_HTTP_QUEUE_H +#define TDENGINE_HTTP_QUEUE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +bool httpInitResultQueue(); +void httpCleanupResultQueue(); +void httpDispatchToResultQueue(); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c new file mode 100644 index 0000000000..9625102f74 --- /dev/null +++ b/src/plugins/http/src/httpQueue.c @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tqueue.h" +#include "tnote.h" +#include "taos.h" +#include "tsclient.h" +#include "httpInt.h" +#include "httpContext.h" +#include "httpSql.h" +#include "httpResp.h" +#include "httpAuth.h" +#include "httpSession.h" + +typedef struct { + pthread_t thread; + int32_t workerId; +} SHttpWorker; + +typedef struct { + int32_t num; + SHttpWorker *httpWorker; +} SHttpWorkerPool; + +typedef struct { + void *param; + void *result; + int numOfRows; + void (*fp)(void *param, void *result, int numOfRows); +} SHttpResult; + +static SHttpWorkerPool tsHttpPool; +static taos_qset tsHttpQset; +static taos_queue tsHttpQueue; + +void httpDispatchToResultQueue(void *param, TAOS_RES *result, int numOfRows, void (*fp)(void *param, void *result, int numOfRows)) { + if (tsHttpQueue != NULL) { + SHttpResult *pMsg = (SHttpResult *)taosAllocateQitem(sizeof(SHttpResult)); + pMsg->param = param; + pMsg->result = result; + pMsg->numOfRows = numOfRows; + pMsg->fp = fp; + taosWriteQitem(tsHttpQueue, TAOS_QTYPE_RPC, pMsg); + } else { + (*fp)(param, result, numOfRows); + } +} + +static void *httpProcessResultQueue(void *param) { + SHttpResult *pMsg; + int32_t type; + void *unUsed; + + while (1) { + if (taosReadQitemFromQset(tsHttpQset, &type, (void **)&pMsg, &unUsed) == 0) { + httpDebug("httpResultQueue: got no message from qset, exiting..."); + break; + } + + httpDebug("context:%p, res:%p will be processed in result queue", pMsg->param, pMsg->result); + (*pMsg->fp)(pMsg->param, pMsg->result, pMsg->numOfRows); + taosFreeQitem(pMsg); + } + + return NULL; +} + +static bool httpAllocateResultQueue() { + tsHttpQueue = taosOpenQueue(); + if (tsHttpQueue == NULL) return false; + + taosAddIntoQset(tsHttpQset, tsHttpQueue, NULL); + + for (int32_t i = 0; i < tsHttpPool.num; ++i) { + SHttpWorker *pWorker = tsHttpPool.httpWorker + i; + pWorker->workerId = i; + + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&pWorker->thread, &thAttr, httpProcessResultQueue, pWorker) != 0) { + httpError("failed to create thread to process http result queue, reason:%s", strerror(errno)); + } + + pthread_attr_destroy(&thAttr); + httpDebug("http result worker:%d is launched, total:%d", pWorker->workerId, tsHttpPool.num); + } + + httpInfo("http result queue is opened"); + return true; +} + +static void httpFreeResultQueue() { + taosCloseQueue(tsHttpQueue); + tsHttpQueue = NULL; +} + +bool httpInitResultQueue() { + tsHttpQset = taosOpenQset(); + + tsHttpPool.num = tsHttpMaxThreads; + tsHttpPool.httpWorker = (SHttpWorker *)calloc(sizeof(SHttpWorker), tsHttpPool.num); + + if (tsHttpPool.httpWorker == NULL) return -1; + for (int32_t i = 0; i < tsHttpPool.num; ++i) { + SHttpWorker *pWorker = tsHttpPool.httpWorker + i; + pWorker->workerId = i; + } + + return httpAllocateResultQueue(); +} + +void httpCleanupResultQueue() { + httpFreeResultQueue(); + + for (int32_t i = 0; i < tsHttpPool.num; ++i) { + SHttpWorker *pWorker = tsHttpPool.httpWorker + i; + if (pWorker->thread) { + taosQsetThreadResume(tsHttpQset); + } + } + + for (int32_t i = 0; i < tsHttpPool.num; ++i) { + SHttpWorker *pWorker = tsHttpPool.httpWorker + i; + if (pWorker->thread) { + pthread_join(pWorker->thread, NULL); + } + } + + taosCloseQset(tsHttpQset); + free(tsHttpPool.httpWorker); + + httpInfo("http result queue is closed"); +} diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 041fbdb92a..07cdea1380 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -24,12 +24,15 @@ #include "httpResp.h" #include "httpAuth.h" #include "httpSession.h" +#include "httpQueue.h" void *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos); void httpProcessMultiSql(HttpContext *pContext); -void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) { +void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows); + +void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int numOfRows) { HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; @@ -75,7 +78,11 @@ void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numO } } -void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int code) { +void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) { + httpDispatchToResultQueue(param, result, numOfRows, httpProcessMultiSqlRetrieveCallBackImp); +} + +void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) { HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; @@ -154,6 +161,10 @@ void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int code) { } } +void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) { + httpDispatchToResultQueue(param, result, unUsedCode, httpProcessMultiSqlCallBackImp); +} + void httpProcessMultiSql(HttpContext *pContext) { HttpSqlCmds * multiCmds = pContext->multiCmds; HttpEncodeMethod *encode = pContext->encodeMethod; @@ -196,7 +207,9 @@ void httpProcessMultiSqlCmd(HttpContext *pContext) { httpProcessMultiSql(pContext); } -void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) { +void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows); + +void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int numOfRows) { HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; @@ -243,7 +256,11 @@ void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int num } } -void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) { +void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) { + httpDispatchToResultQueue(param, result, numOfRows, httpProcessSingleSqlRetrieveCallBackImp); +} + +void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCode) { HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; @@ -306,6 +323,10 @@ void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) } } +void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) { + httpDispatchToResultQueue(param, result, unUsedCode, httpProcessSingleSqlCallBackImp); +} + void httpProcessSingleSqlCmd(HttpContext *pContext) { HttpSqlCmd * cmd = &pContext->singleCmd; char * sql = cmd->nativSql; diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index 38bd8624b2..e51c8dd4f7 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -26,6 +26,7 @@ #include "httpServer.h" #include "httpResp.h" #include "httpHandle.h" +#include "httpQueue.h" #include "gcHandle.h" #include "restHandle.h" #include "tgHandle.h" @@ -67,6 +68,11 @@ int httpStartSystem() { return -1; } + if (!httpInitResultQueue()) { + httpError("http init result queue failed"); + return -1; + } + if (!httpInitContexts()) { httpError("http init contexts failed"); return -1; @@ -98,6 +104,8 @@ void httpCleanUpSystem() { httpCleanUpConnect(); httpCleanupContexts(); httpCleanUpSessions(); + httpCleanupResultQueue(); + pthread_mutex_destroy(&tsHttpServer.serverMutex); taosTFree(tsHttpServer.pThreads); tsHttpServer.pThreads = NULL; diff --git a/src/query/inc/qPercentile.h b/src/query/inc/qPercentile.h index 52f666c338..0a52d4f205 100644 --- a/src/query/inc/qPercentile.h +++ b/src/query/inc/qPercentile.h @@ -17,6 +17,8 @@ #define TDENGINE_QPERCENTILE_H #include "qExtbuffer.h" +#include "qResultbuf.h" +#include "qTsbuf.h" typedef struct MinMaxEntry { union { @@ -31,47 +33,43 @@ typedef struct MinMaxEntry { }; } MinMaxEntry; -typedef struct tMemBucketSegment { - int32_t numOfSlots; - MinMaxEntry * pBoundingEntries; - tExtMemBuffer **pBuffer; -} tMemBucketSegment; +typedef struct { + int32_t size; + int32_t pageId; + tFilePage *data; +} SSlotInfo; + +typedef struct tMemBucketSlot { + SSlotInfo info; + MinMaxEntry range; +} tMemBucketSlot; + +struct tMemBucket; +typedef int32_t (*__perc_hash_func_t)(struct tMemBucket *pBucket, const void *value); typedef struct tMemBucket { - int16_t numOfSegs; - int16_t nTotalSlots; - int16_t nSlotsOfSeg; - int16_t dataType; - - int16_t nElemSize; - int32_t numOfElems; - - int32_t nTotalBufferSize; - int32_t maxElemsCapacity; - - int32_t pageSize; - int16_t numOfTotalPages; - int16_t numOfAvailPages; /* remain available buffer pages */ - - tMemBucketSegment *pSegs; - tOrderDescriptor * pOrderDesc; - - MinMaxEntry nRange; - - void (*HashFunc)(struct tMemBucket *pBucket, void *value, int16_t *segIdx, int16_t *slotIdx); + int16_t numOfSlots; + int16_t type; + int16_t bytes; + int32_t total; + int32_t elemPerPage; // number of elements for each object + int32_t maxCapacity; // maximum allowed number of elements that can be sort directly to get the result + int32_t bufPageSize; // disk page size + MinMaxEntry range; // value range + int32_t times; // count that has been checked for deciding the correct data value buckets. + __compar_fn_t comparFn; + + tMemBucketSlot *pSlots; + SDiskbasedResultBuf *pBuffer; + __perc_hash_func_t hashFunc; } tMemBucket; -tMemBucket *tMemBucketCreate(int32_t totalSlots, int32_t nBufferSize, int16_t nElemSize, int16_t dataType, - tOrderDescriptor *pDesc); +tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType); void tMemBucketDestroy(tMemBucket *pBucket); -void tMemBucketPut(tMemBucket *pBucket, void *data, int32_t numOfRows); +void tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size); double getPercentile(tMemBucket *pMemBucket, double percent); -void tBucketIntHash(tMemBucket *pBucket, void *value, int16_t *segIdx, int16_t *slotIdx); - -void tBucketDoubleHash(tMemBucket *pBucket, void *value, int16_t *segIdx, int16_t *slotIdx); - #endif // TDENGINE_QPERCENTILE_H diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 65ab82883b..384d8079a7 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -168,6 +168,7 @@ typedef struct SQLFunctionCtx { int16_t outputType; int16_t outputBytes; // size of results, determined by function and input column data type bool hasNull; // null value exist in current block + bool requireNull; // require null in some function int16_t functionId; // function id void * aInputElemBuf; char * aOutputBuf; // final result output buffer, point to sdata->data diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index e32991d504..78632023f3 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -35,9 +35,7 @@ * forced to load primary column explicitly. */ #define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0) -#define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) -#define TSDB_COL_IS_NORMAL_COL(f) ((f) == TSDB_COL_NORMAL) -#define TSDB_COL_IS_UD_COL(f) ((f) == TSDB_COL_UDC) + #define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) @@ -157,12 +155,12 @@ static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) { time_t t = (time_t)key; localtime_r(&t, &tm); - int mon = tm.tm_year * 12 + tm.tm_mon + interval * factor; + int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); tm.tm_year = mon / 12; tm.tm_mon = mon % 12; tw->skey = mktime(&tm) * 1000L; - mon += interval; + mon = (int)(mon + interval); tm.tm_year = mon / 12; tm.tm_mon = mon % 12; tw->ekey = mktime(&tm) * 1000L; @@ -285,7 +283,7 @@ bool isGroupbyNormalCol(SSqlGroupbyExpr *pGroupbyExpr) { for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { SColIndex *pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, i); - if (pColIndex->flag == TSDB_COL_NORMAL) { + if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { //make sure the normal column locates at the second position if tbname exists in group by clause if (pGroupbyExpr->numOfGroupCols > 1) { assert(pColIndex->colIndex > 0); @@ -306,7 +304,7 @@ int16_t getGroupbyColumnType(SQuery *pQuery, SSqlGroupbyExpr *pGroupbyExpr) { for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { SColIndex *pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, i); - if (pColIndex->flag == TSDB_COL_NORMAL) { + if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { colId = pColIndex->colId; break; } @@ -1133,7 +1131,7 @@ static char *getGroupbyColumnData(SQuery *pQuery, int16_t *type, int16_t *bytes, for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); - if (pColIndex->flag == TSDB_COL_TAG) { + if (TSDB_COL_IS_TAG(pColIndex->flag)) { continue; } @@ -1603,6 +1601,13 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; SColIndex* pIndex = &pSqlFuncMsg->colInfo; + if (TSDB_COL_REQ_NULL(pIndex->flag)) { + pCtx->requireNull = true; + pIndex->flag &= ~(TSDB_COL_NULL); + } else { + pCtx->requireNull = false; + } + int32_t index = pSqlFuncMsg->colInfo.colIndex; if (TSDB_COL_IS_TAG(pIndex->flag)) { if (pIndex->colId == TSDB_TBNAME_COLUMN_INDEX) { // todo refactor @@ -1622,6 +1627,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->inputType = pQuery->colList[index].type; } + assert(isValidDataType(pCtx->inputType)); pCtx->ptsOutputBuf = NULL; @@ -1831,7 +1837,7 @@ static bool onlyQueryTags(SQuery* pQuery) { if (functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TID_TAG && (!(functionId == TSDB_FUNC_COUNT && pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX)) && - (!(functionId == TSDB_FUNC_PRJ && pExprInfo->base.colInfo.flag == TSDB_COL_UDC))) { + (!(functionId == TSDB_FUNC_PRJ && TSDB_COL_IS_UD_COL(pExprInfo->base.colInfo.flag)))) { return false; } } @@ -1958,6 +1964,15 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, bool stableQuery) { return; } + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + pQuery->order.order = TSDB_ORDER_ASC; + if (pQuery->window.skey > pQuery->window.ekey) { + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } + + return; + } + if (isPointInterpoQuery(pQuery) && pQuery->intervalTime == 0) { if (!QUERY_IS_ASC_QUERY(pQuery)) { qDebug(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, @@ -2119,35 +2134,36 @@ static bool needToLoadDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDat return false; } -#define PT_IN_WINDOW(_p, _w) ((_p) > (_w).skey && (_p) < (_w).ekey) - static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) { STimeWindow w = {0}; TSKEY sk = MIN(pQuery->window.skey, pQuery->window.ekey); TSKEY ek = MAX(pQuery->window.skey, pQuery->window.ekey); - if (QUERY_IS_ASC_QUERY(pQuery)) { getAlignQueryTimeWindow(pQuery, pBlockInfo->window.skey, sk, ek, &w); + assert(w.ekey >= pBlockInfo->window.skey); - if (PT_IN_WINDOW(w.ekey, pBlockInfo->window)) { + if (w.ekey < pBlockInfo->window.ekey) { return true; } while(1) { GET_NEXT_TIMEWINDOW(pQuery, &w); - if (w.skey > pBlockInfo->window.skey) { + if (w.skey > pBlockInfo->window.ekey) { break; } - if (PT_IN_WINDOW(w.skey, pBlockInfo->window) || PT_IN_WINDOW(w.ekey, pBlockInfo->window)) { + assert(w.ekey > pBlockInfo->window.ekey); + if (w.skey <= pBlockInfo->window.ekey && w.skey > pBlockInfo->window.skey) { return true; } } } else { getAlignQueryTimeWindow(pQuery, pBlockInfo->window.ekey, sk, ek, &w); - if (PT_IN_WINDOW(w.skey, pBlockInfo->window)) { + assert(w.skey <= pBlockInfo->window.ekey); + + if (w.skey > pBlockInfo->window.skey) { return true; } @@ -2157,7 +2173,8 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) { break; } - if (PT_IN_WINDOW(w.skey, pBlockInfo->window) || PT_IN_WINDOW(w.ekey, pBlockInfo->window)) { + assert(w.skey < pBlockInfo->window.skey); + if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) { return true; } } @@ -4434,7 +4451,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo // NOTE: pTableCheckInfo need to update the query time range and the lastKey info // TODO fixme - changeExecuteScanOrder(pQInfo, false); + changeExecuteScanOrder(pQInfo, isSTableQuery); code = setupQueryHandle(tsdb, pQInfo, isSTableQuery); if (code != TSDB_CODE_SUCCESS) { @@ -5402,7 +5419,7 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE j += 1; } - } else if (pExprMsg->colInfo.flag == TSDB_COL_UDC) { // user specified column data + } else if (TSDB_COL_IS_UD_COL(pExprMsg->colInfo.flag)) { // user specified column data return TSDB_UD_COLUMN_INDEX; } else { while (j < pQueryMsg->numOfCols) { @@ -5610,7 +5627,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, int16_t functionId = pExprMsg->functionId; if (functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TAG_DUMMY) { - if (pExprMsg->colInfo.flag != TSDB_COL_TAG) { // ignore the column index check for arithmetic expression. + if (!TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { // ignore the column index check for arithmetic expression. code = TSDB_CODE_QRY_INVALID_MSG; goto _cleanup; } @@ -6143,6 +6160,9 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, goto _cleanup; } + // NOTE: pTableCheckInfo need to update the query time range and the lastKey info +// changeExecuteScanOrder(pQInfo, stableQuery); + int32_t index = 0; for(int32_t i = 0; i < numOfGroups; ++i) { @@ -6893,7 +6913,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) { int16_t type = 0, bytes = 0; for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { // not assign value in case of user defined constant output column - if (pExprInfo[j].base.colInfo.flag == TSDB_COL_UDC) { + if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.colInfo.flag)) { continue; } diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index b3bb443fd8..ddb63c5012 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -55,7 +55,7 @@ SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_ SFillColInfo* pColInfo = &pFillInfo->pFillCol[i]; pFillInfo->pData[i] = calloc(1, pColInfo->col.bytes * capacity); - if (pColInfo->flag == TSDB_COL_TAG) { + if (TSDB_COL_IS_TAG(pColInfo->flag)) { bool exists = false; for(int32_t j = 0; j < k; ++j) { if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) { @@ -155,7 +155,7 @@ void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInpu char* data = pInput->data + pCol->col.offset * pInput->num; memcpy(pFillInfo->pData[i], data, (size_t)(pInput->num * pCol->col.bytes)); - if (pCol->flag == TSDB_COL_TAG) { // copy the tag value to tag value buffer + if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer for (int32_t j = 0; j < pFillInfo->numOfTags; ++j) { SFillTagColInfo* pTag = &pFillInfo->pTags[j]; if (pTag->col.colId == pCol->col.colId) { @@ -259,7 +259,7 @@ int taosDoLinearInterpolation(int32_t type, SPoint* point1, SPoint* point2, SPoi static void setTagsValue(SFillInfo* pFillInfo, tFilePage** data, int32_t num) { for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) { SFillColInfo* pCol = &pFillInfo->pFillCol[j]; - if (pCol->flag == TSDB_COL_NORMAL) { + if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) { continue; } @@ -459,7 +459,7 @@ int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t nu // assign rows to dst buffer for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (pCol->flag == TSDB_COL_TAG) { + if (TSDB_COL_IS_TAG(pCol->flag)) { continue; } diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c index 19775075fc..3e9b077d30 100644 --- a/src/query/src/qPercentile.c +++ b/src/query/src/qPercentile.c @@ -14,310 +14,291 @@ */ #include "qPercentile.h" +#include "qResultbuf.h" #include "os.h" #include "queryLog.h" #include "taosdef.h" -#include "taosmsg.h" #include "tulog.h" +#include "tcompare.h" -tExtMemBuffer *releaseBucketsExceptFor(tMemBucket *pMemBucket, int16_t segIdx, int16_t slotIdx) { - tExtMemBuffer *pBuffer = NULL; - - for (int32_t i = 0; i < pMemBucket->numOfSegs; ++i) { - tMemBucketSegment *pSeg = &pMemBucket->pSegs[i]; - - for (int32_t j = 0; j < pSeg->numOfSlots; ++j) { - if (i == segIdx && j == slotIdx) { - pBuffer = pSeg->pBuffer[j]; - } else { - if (pSeg->pBuffer && pSeg->pBuffer[j]) { - pSeg->pBuffer[j] = destoryExtMemBuffer(pSeg->pBuffer[j]); - } - } - } - } - - return pBuffer; +#define DEFAULT_NUM_OF_SLOT 1024 + +int32_t getGroupId(int32_t numOfSlots, int32_t slotIndex, int32_t times) { + return (times * numOfSlots) + slotIndex; } -static tFilePage *loadIntoBucketFromDisk(tMemBucket *pMemBucket, int32_t segIdx, int32_t slotIdx, - tOrderDescriptor *pDesc) { - // release all data in other slots - tExtMemBuffer *pMemBuffer = pMemBucket->pSegs[segIdx].pBuffer[slotIdx]; - tFilePage * buffer = (tFilePage *)calloc(1, pMemBuffer->nElemSize * pMemBuffer->numOfTotalElems + sizeof(tFilePage)); - int32_t oldCapacity = pDesc->pColumnModel->capacity; - pDesc->pColumnModel->capacity = pMemBuffer->numOfTotalElems; - - if (!tExtMemBufferIsAllDataInMem(pMemBuffer)) { - pMemBuffer = releaseBucketsExceptFor(pMemBucket, segIdx, slotIdx); - assert(pMemBuffer->numOfTotalElems > 0); - - // load data in disk to memory - tFilePage *pPage = (tFilePage *)calloc(1, pMemBuffer->pageSize); - - for (uint32_t i = 0; i < pMemBuffer->fileMeta.flushoutData.nLength; ++i) { - tFlushoutInfo *pFlushInfo = &pMemBuffer->fileMeta.flushoutData.pFlushoutInfo[i]; - - int32_t ret = fseek(pMemBuffer->file, pFlushInfo->startPageId * pMemBuffer->pageSize, SEEK_SET); - UNUSED(ret); - - for (uint32_t j = 0; j < pFlushInfo->numOfPages; ++j) { - ret = (int32_t)fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file); - UNUSED(ret); - assert(pPage->num > 0); - - tColModelAppend(pDesc->pColumnModel, buffer, pPage->data, 0, (int32_t)pPage->num, (int32_t)pPage->num); - printf("id: %d count: %" PRIu64 "\n", j, buffer->num); - } - } - taosTFree(pPage); - - assert(buffer->num == pMemBuffer->fileMeta.numOfElemsInFile); +static tFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) { + tFilePage *buffer = (tFilePage *)calloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(tFilePage)); + + int32_t groupId = getGroupId(pMemBucket->numOfSlots, slotIdx, pMemBucket->times); + SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId); + + int32_t offset = 0; + for(int32_t i = 0; i < list->size; ++i) { + SPageInfo* pgInfo = *(SPageInfo**) taosArrayGet(list, i); + + tFilePage* pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId); + memcpy(buffer->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes)); + + offset += (int32_t)(pg->num * pMemBucket->bytes); } - - // load data in pMemBuffer to buffer - tFilePagesItem *pListItem = pMemBuffer->pHead; - while (pListItem != NULL) { - tColModelAppend(pDesc->pColumnModel, buffer, pListItem->item.data, 0, (int32_t)pListItem->item.num, - (int32_t)pListItem->item.num); - pListItem = pListItem->pNext; - } - - tColDataQSort(pDesc, (int32_t)buffer->num, 0, (int32_t)buffer->num - 1, buffer->data, TSDB_ORDER_ASC); - - pDesc->pColumnModel->capacity = oldCapacity; // restore value + + qsort(buffer->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn); return buffer; } -double findOnlyResult(tMemBucket *pMemBucket) { - assert(pMemBucket->numOfElems == 1); - - for (int32_t i = 0; i < pMemBucket->numOfSegs; ++i) { - tMemBucketSegment *pSeg = &pMemBucket->pSegs[i]; - if (pSeg->pBuffer) { - for (int32_t j = 0; j < pSeg->numOfSlots; ++j) { - tExtMemBuffer *pBuffer = pSeg->pBuffer[j]; - if (pBuffer) { - assert(pBuffer->numOfTotalElems == 1); - tFilePage *pPage = &pBuffer->pHead->item; - if (pBuffer->numOfElemsInBuffer == 1) { - switch (pMemBucket->dataType) { - case TSDB_DATA_TYPE_INT: - return *(int32_t *)pPage->data; - case TSDB_DATA_TYPE_SMALLINT: - return *(int16_t *)pPage->data; - case TSDB_DATA_TYPE_TINYINT: - return *(int8_t *)pPage->data; - case TSDB_DATA_TYPE_BIGINT: - return (double)(*(int64_t *)pPage->data); - case TSDB_DATA_TYPE_DOUBLE: { - double dv = GET_DOUBLE_VAL(pPage->data); - //return *(double *)pPage->data; - return dv; - } - case TSDB_DATA_TYPE_FLOAT: { - float fv = GET_FLOAT_VAL(pPage->data); - //return *(float *)pPage->data; - return fv; - } - default: - return 0; - } - } - } - } - } - } - return 0; -} - -void tBucketBigIntHash(tMemBucket *pBucket, void *value, int16_t *segIdx, int16_t *slotIdx) { - int64_t v = *(int64_t *)value; - - if (pBucket->nRange.i64MaxVal == INT64_MIN) { - if (v >= 0) { - *segIdx = ((v >> (64 - 9)) >> 6) + 8; - *slotIdx = (v >> (64 - 9)) & 0x3F; - } else { // v<0 - *segIdx = ((-v) >> (64 - 9)) >> 6; - *slotIdx = ((-v) >> (64 - 9)) & 0x3F; - *segIdx = 7 - (*segIdx); - } - } else { - // todo hash for bigint and float and double - int64_t span = pBucket->nRange.i64MaxVal - pBucket->nRange.i64MinVal; - if (span < pBucket->nTotalSlots) { - int32_t delta = (int32_t)(v - pBucket->nRange.i64MinVal); - *segIdx = delta / pBucket->nSlotsOfSeg; - *slotIdx = delta % pBucket->nSlotsOfSeg; - } else { - double x = (double)span / pBucket->nTotalSlots; - double posx = (v - pBucket->nRange.i64MinVal) / x; - if (v == pBucket->nRange.i64MaxVal) { - posx -= 1; - } - - *segIdx = ((int32_t)posx) / pBucket->nSlotsOfSeg; - *slotIdx = ((int32_t)posx) % pBucket->nSlotsOfSeg; - } - } -} - -// todo refactor to more generic -void tBucketIntHash(tMemBucket *pBucket, void *value, int16_t *segIdx, int16_t *slotIdx) { - int32_t v = *(int32_t *)value; - - if (pBucket->nRange.iMaxVal == INT32_MIN) { - /* - * taking negative integer into consideration, - * there is only half of pBucket->segs available for non-negative integer - */ - // int32_t numOfSlots = pBucket->nTotalSlots>>1; - // int32_t bits = bitsOfNumber(numOfSlots)-1; - - if (v >= 0) { - *segIdx = ((v >> (32 - 9)) >> 6) + 8; - *slotIdx = (v >> (32 - 9)) & 0x3F; - } else { // v<0 - *segIdx = ((-v) >> (32 - 9)) >> 6; - *slotIdx = ((-v) >> (32 - 9)) & 0x3F; - *segIdx = 7 - (*segIdx); - } - } else { - // divide a range of [iMinVal, iMaxVal] into 1024 buckets - int32_t span = pBucket->nRange.iMaxVal - pBucket->nRange.iMinVal; - if (span < pBucket->nTotalSlots) { - int32_t delta = v - pBucket->nRange.iMinVal; - *segIdx = delta / pBucket->nSlotsOfSeg; - *slotIdx = delta % pBucket->nSlotsOfSeg; - } else { - double x = (double)span / pBucket->nTotalSlots; - double posx = (v - pBucket->nRange.iMinVal) / x; - if (v == pBucket->nRange.iMaxVal) { - posx -= 1; - } - *segIdx = ((int32_t)posx) / pBucket->nSlotsOfSeg; - *slotIdx = ((int32_t)posx) % pBucket->nSlotsOfSeg; - } - } -} - -void tBucketDoubleHash(tMemBucket *pBucket, void *value, int16_t *segIdx, int16_t *slotIdx) { - // double v = *(double *)value; - double v = GET_DOUBLE_VAL(value); - - if (pBucket->nRange.dMinVal == DBL_MAX) { - /* - * taking negative integer into consideration, - * there is only half of pBucket->segs available for non-negative integer - */ - double x = DBL_MAX / (pBucket->nTotalSlots >> 1); - double posx = (v + DBL_MAX) / x; - *segIdx = ((int32_t)posx) / pBucket->nSlotsOfSeg; - *slotIdx = ((int32_t)posx) % pBucket->nSlotsOfSeg; - } else { - // divide a range of [dMinVal, dMaxVal] into 1024 buckets - double span = pBucket->nRange.dMaxVal - pBucket->nRange.dMinVal; - if (span < pBucket->nTotalSlots) { - int32_t delta = (int32_t)(v - pBucket->nRange.dMinVal); - *segIdx = delta / pBucket->nSlotsOfSeg; - *slotIdx = delta % pBucket->nSlotsOfSeg; - } else { - double x = span / pBucket->nTotalSlots; - double posx = (v - pBucket->nRange.dMinVal) / x; - if (v == pBucket->nRange.dMaxVal) { - posx -= 1; - } - *segIdx = ((int32_t)posx) / pBucket->nSlotsOfSeg; - *slotIdx = ((int32_t)posx) % pBucket->nSlotsOfSeg; - } - - if (*segIdx < 0 || *segIdx > 16 || *slotIdx < 0 || *slotIdx > 64) { - uError("error in hash process. segment is: %d, slot id is: %d\n", *segIdx, *slotIdx); - } - } -} - -tMemBucket *tMemBucketCreate(int32_t totalSlots, int32_t nBufferSize, int16_t nElemSize, int16_t dataType, - tOrderDescriptor *pDesc) { - tMemBucket *pBucket = (tMemBucket *)malloc(sizeof(tMemBucket)); - pBucket->nTotalSlots = totalSlots; - pBucket->nSlotsOfSeg = 1 << 6; // 64 Segments, 16 slots each seg. - pBucket->dataType = dataType; - pBucket->nElemSize = nElemSize; - pBucket->pageSize = DEFAULT_PAGE_SIZE; - - pBucket->numOfElems = 0; - pBucket->numOfSegs = pBucket->nTotalSlots / pBucket->nSlotsOfSeg; - - pBucket->nTotalBufferSize = nBufferSize; - - pBucket->maxElemsCapacity = pBucket->nTotalBufferSize / pBucket->nElemSize; - - pBucket->numOfTotalPages = pBucket->nTotalBufferSize / pBucket->pageSize; - pBucket->numOfAvailPages = pBucket->numOfTotalPages; - - pBucket->pSegs = NULL; - pBucket->pOrderDesc = pDesc; - - switch (pBucket->dataType) { +static void resetBoundingBox(MinMaxEntry* range, int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_BIGINT: { + range->i64MaxVal = INT64_MIN; + range->i64MinVal = INT64_MAX; + break; + }; case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_TINYINT: { - pBucket->nRange.iMinVal = INT32_MAX; - pBucket->nRange.iMaxVal = INT32_MIN; - pBucket->HashFunc = tBucketIntHash; + range->iMaxVal = INT32_MIN; + range->iMinVal = INT32_MAX; break; }; case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_FLOAT: { - pBucket->nRange.dMinVal = DBL_MAX; - pBucket->nRange.dMaxVal = -DBL_MAX; - pBucket->HashFunc = tBucketDoubleHash; + range->dMaxVal = -DBL_MAX; + range->dMinVal = DBL_MAX; break; - }; - case TSDB_DATA_TYPE_BIGINT: { - pBucket->nRange.i64MinVal = INT64_MAX; - pBucket->nRange.i64MaxVal = INT64_MIN; - pBucket->HashFunc = tBucketBigIntHash; - break; - }; - default: { - uError("MemBucket:%p,not support data type %d,failed", pBucket, pBucket->dataType); - taosTFree(pBucket); - return NULL; + } + } +} + +static void resetPosInfo(SSlotInfo* pInfo) { + pInfo->size = 0; + pInfo->pageId = -1; + pInfo->data = NULL; +} + +double findOnlyResult(tMemBucket *pMemBucket) { + assert(pMemBucket->total == 1); + + for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) { + tMemBucketSlot *pSlot = &pMemBucket->pSlots[i]; + if (pSlot->info.size == 0) { + continue; + } + + int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times); + SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId); + assert(list->size == 1); + + SPageInfo* pgInfo = (SPageInfo*) taosArrayGetP(list, 0); + tFilePage* pPage = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId); + assert(pPage->num == 1); + + switch (pMemBucket->type) { + case TSDB_DATA_TYPE_INT: + return *(int32_t *)pPage->data; + case TSDB_DATA_TYPE_SMALLINT: + return *(int16_t *)pPage->data; + case TSDB_DATA_TYPE_TINYINT: + return *(int8_t *)pPage->data; + case TSDB_DATA_TYPE_BIGINT: + return (double)(*(int64_t *)pPage->data); + case TSDB_DATA_TYPE_DOUBLE: { + double dv = GET_DOUBLE_VAL(pPage->data); + return dv; + } + case TSDB_DATA_TYPE_FLOAT: { + float fv = GET_FLOAT_VAL(pPage->data); + return fv; + } + default: + return 0; } } - int32_t numOfCols = pDesc->pColumnModel->numOfCols; - if (numOfCols != 1) { - uError("MemBucket:%p,only consecutive data is allowed,invalid numOfCols:%d", pBucket, numOfCols); - taosTFree(pBucket); + return 0; +} + +int32_t tBucketBigIntHash(tMemBucket *pBucket, const void *value) { + int64_t v = *(int64_t *)value; + int32_t index = -1; + + int32_t halfSlot = pBucket->numOfSlots >> 1; +// int32_t bits = 32;//bitsOfNumber(pBucket->numOfSlots) - 1; + + if (pBucket->range.i64MaxVal == INT64_MIN) { + if (v >= 0) { + index = (v >> (64 - 9)) + halfSlot; + } else { // v<0 + index = ((-v) >> (64 - 9)); + index = -index + (halfSlot - 1); + } + + return index; + } else { + // todo hash for bigint and float and double + int64_t span = pBucket->range.i64MaxVal - pBucket->range.i64MinVal; + if (span < pBucket->numOfSlots) { + int32_t delta = (int32_t)(v - pBucket->range.i64MinVal); + index = delta % pBucket->numOfSlots; + } else { + double slotSpan = (double)span / pBucket->numOfSlots; + index = (int32_t)((v - pBucket->range.i64MinVal) / slotSpan); + if (v == pBucket->range.i64MaxVal) { + index -= 1; + } + } + + return index; + } +} + +// todo refactor to more generic +int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { + int32_t v = *(int32_t *)value; + int32_t index = -1; + + if (pBucket->range.iMaxVal == INT32_MIN) { + /* + * taking negative integer into consideration, + * there is only half of pBucket->segs available for non-negative integer + */ + int32_t halfSlot = pBucket->numOfSlots >> 1; + int32_t bits = 32;//bitsOfNumber(pBucket->numOfSlots) - 1; + + if (v >= 0) { + index = (v >> (bits - 9)) + halfSlot; + } else { // v < 0 + index = ((-v) >> (32 - 9)); + index = -index + (halfSlot - 1); + } + + return index; + } else { + // divide a range of [iMinVal, iMaxVal] into 1024 buckets + int32_t span = pBucket->range.iMaxVal - pBucket->range.iMinVal; + if (span < pBucket->numOfSlots) { + int32_t delta = v - pBucket->range.iMinVal; + index = (delta % pBucket->numOfSlots); + } else { + double slotSpan = (double)span / pBucket->numOfSlots; + index = (int32_t)((v - pBucket->range.iMinVal) / slotSpan); + if (v == pBucket->range.iMaxVal) { + index -= 1; + } + } + + return index; + } +} + +int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) { + double v = GET_DOUBLE_VAL(value); + int32_t index = -1; + + if (pBucket->range.dMinVal == DBL_MAX) { + /* + * taking negative integer into consideration, + * there is only half of pBucket->segs available for non-negative integer + */ + double x = DBL_MAX / (pBucket->numOfSlots >> 1); + double posx = (v + DBL_MAX) / x; + return ((int32_t)posx) % pBucket->numOfSlots; + } else { + // divide a range of [dMinVal, dMaxVal] into 1024 buckets + double span = pBucket->range.dMaxVal - pBucket->range.dMinVal; + if (span < pBucket->numOfSlots) { + int32_t delta = (int32_t)(v - pBucket->range.dMinVal); + index = (delta % pBucket->numOfSlots); + } else { + double slotSpan = span / pBucket->numOfSlots; + index = (int32_t)((v - pBucket->range.dMinVal) / slotSpan); + if (v == pBucket->range.dMaxVal) { + index -= 1; + } + } + + if (index < 0 || index > pBucket->numOfSlots) { + uError("error in hash process. slot id: %d", index); + } + + return index; + } +} + +static __perc_hash_func_t getHashFunc(int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_TINYINT: { + return tBucketIntHash; + }; + + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_FLOAT: { + return tBucketDoubleHash; + }; + + case TSDB_DATA_TYPE_BIGINT: { + return tBucketBigIntHash; + }; + + default: { + return NULL; + } + } +} + +static void resetSlotInfo(tMemBucket* pBucket) { + for (int32_t i = 0; i < pBucket->numOfSlots; ++i) { + tMemBucketSlot* pSlot = &pBucket->pSlots[i]; + + resetBoundingBox(&pSlot->range, pBucket->type); + resetPosInfo(&pSlot->info); + } +} + +tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType) { + tMemBucket *pBucket = (tMemBucket *)calloc(1, sizeof(tMemBucket)); + if (pBucket == NULL) { return NULL; } - SSchema* pSchema = getColumnModelSchema(pDesc->pColumnModel, 0); - if (pSchema->type != dataType) { - uError("MemBucket:%p,data type is not consistent,%d in schema, %d in param", pBucket, pSchema->type, dataType); - taosTFree(pBucket); + pBucket->numOfSlots = DEFAULT_NUM_OF_SLOT; + pBucket->bufPageSize = DEFAULT_PAGE_SIZE * 4; // 4k per page + + pBucket->type = dataType; + pBucket->bytes = nElemSize; + pBucket->total = 0; + pBucket->times = 1; + + pBucket->maxCapacity = 200000; + + pBucket->elemPerPage = (pBucket->bufPageSize - sizeof(tFilePage))/pBucket->bytes; + pBucket->comparFn = getKeyComparFunc(pBucket->type); + resetBoundingBox(&pBucket->range, pBucket->type); + + pBucket->hashFunc = getHashFunc(pBucket->type); + if (pBucket->hashFunc == NULL) { + uError("MemBucket:%p, not support data type %d, failed", pBucket, pBucket->type); + free(pBucket); return NULL; } - if (pBucket->numOfTotalPages < pBucket->nTotalSlots) { - uWarn("MemBucket:%p,total buffer pages %d are not enough for all slots", pBucket, pBucket->numOfTotalPages); + pBucket->pSlots = (tMemBucketSlot *)calloc(pBucket->numOfSlots, sizeof(tMemBucketSlot)); + if (pBucket->pSlots == NULL) { + free(pBucket); + return NULL; } - pBucket->pSegs = (tMemBucketSegment *)malloc(pBucket->numOfSegs * sizeof(tMemBucketSegment)); + resetSlotInfo(pBucket); - for (int32_t i = 0; i < pBucket->numOfSegs; ++i) { - pBucket->pSegs[i].numOfSlots = pBucket->nSlotsOfSeg; - pBucket->pSegs[i].pBuffer = NULL; - pBucket->pSegs[i].pBoundingEntries = NULL; + int32_t ret = createDiskbasedResultBuffer(&pBucket->pBuffer, pBucket->bytes, pBucket->bufPageSize, pBucket->bufPageSize * 512, NULL); + if (ret != TSDB_CODE_SUCCESS) { + tMemBucketDestroy(pBucket); + return NULL; } - - uDebug("MemBucket:%p,created,buffer size:%ld,elem size:%d", pBucket, pBucket->numOfTotalPages * DEFAULT_PAGE_SIZE, - pBucket->nElemSize); - + + uDebug("MemBucket:%p, elem size:%d", pBucket, pBucket->bytes); return pBucket; } @@ -326,81 +307,11 @@ void tMemBucketDestroy(tMemBucket *pBucket) { return; } - if (pBucket->pSegs) { - for (int32_t i = 0; i < pBucket->numOfSegs; ++i) { - tMemBucketSegment *pSeg = &(pBucket->pSegs[i]); - taosTFree(pSeg->pBoundingEntries); - - if (pSeg->pBuffer == NULL || pSeg->numOfSlots == 0) { - continue; - } - - for (int32_t j = 0; j < pSeg->numOfSlots; ++j) { - if (pSeg->pBuffer[j] != NULL) { - pSeg->pBuffer[j] = destoryExtMemBuffer(pSeg->pBuffer[j]); - } - } - taosTFree(pSeg->pBuffer); - } - } - - taosTFree(pBucket->pSegs); + destroyResultBuf(pBucket->pBuffer); + taosTFree(pBucket->pSlots); taosTFree(pBucket); } -/* - * find the slots which accounts for largest proportion of total in-memory buffer - */ -static void tBucketGetMaxMemSlot(tMemBucket *pBucket, int16_t *segIdx, int16_t *slotIdx) { - *segIdx = -1; - *slotIdx = -1; - - int32_t val = 0; - for (int32_t k = 0; k < pBucket->numOfSegs; ++k) { - tMemBucketSegment *pSeg = &pBucket->pSegs[k]; - for (int32_t i = 0; i < pSeg->numOfSlots; ++i) { - if (pSeg->pBuffer == NULL || pSeg->pBuffer[i] == NULL) { - continue; - } - - if (val < pSeg->pBuffer[i]->numOfInMemPages) { - val = pSeg->pBuffer[i]->numOfInMemPages; - *segIdx = k; - *slotIdx = i; - } - } - } -} - -static void resetBoundingBox(tMemBucketSegment *pSeg, int32_t type) { - switch (type) { - case TSDB_DATA_TYPE_BIGINT: { - for (int32_t i = 0; i < pSeg->numOfSlots; ++i) { - pSeg->pBoundingEntries[i].i64MaxVal = INT64_MIN; - pSeg->pBoundingEntries[i].i64MinVal = INT64_MAX; - } - break; - }; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_TINYINT: { - for (int32_t i = 0; i < pSeg->numOfSlots; ++i) { - pSeg->pBoundingEntries[i].iMaxVal = INT32_MIN; - pSeg->pBoundingEntries[i].iMinVal = INT32_MAX; - } - break; - }; - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_FLOAT: { - for (int32_t i = 0; i < pSeg->numOfSlots; ++i) { - pSeg->pBoundingEntries[i].dMaxVal = -DBL_MAX; - pSeg->pBoundingEntries[i].dMinVal = DBL_MAX; - } - break; - } - } -} - void tMemBucketUpdateBoundingBox(MinMaxEntry *r, char *data, int32_t dataType) { switch (dataType) { case TSDB_DATA_TYPE_INT: { @@ -461,7 +372,6 @@ void tMemBucketUpdateBoundingBox(MinMaxEntry *r, char *data, int32_t dataType) { break; }; case TSDB_DATA_TYPE_FLOAT: { - // double val = *(float *)data; double val = GET_FLOAT_VAL(data); if (r->dMinVal > val) { @@ -478,171 +388,95 @@ void tMemBucketUpdateBoundingBox(MinMaxEntry *r, char *data, int32_t dataType) { } /* - * in memory bucket, we only accept the simple data consecutive put in a row/column - * no column-model in this case. + * in memory bucket, we only accept data array list */ -void tMemBucketPut(tMemBucket *pBucket, void *data, int32_t numOfRows) { - pBucket->numOfElems += numOfRows; - int16_t segIdx = 0, slotIdx = 0; +void tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { + assert(pBucket != NULL && data != NULL && size > 0); + pBucket->total += (int32_t)size; - for (int32_t i = 0; i < numOfRows; ++i) { - char *d = (char *)data + i * tDataTypeDesc[pBucket->dataType].nSize; + int32_t bytes = pBucket->bytes; - switch (pBucket->dataType) { - case TSDB_DATA_TYPE_SMALLINT: { - int32_t val = *(int16_t *)d; - (pBucket->HashFunc)(pBucket, &val, &segIdx, &slotIdx); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - int32_t val = *(int8_t *)d; - (pBucket->HashFunc)(pBucket, &val, &segIdx, &slotIdx); - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t val = *(int32_t *)d; - (pBucket->HashFunc)(pBucket, &val, &segIdx, &slotIdx); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t val = *(int64_t *)d; - (pBucket->HashFunc)(pBucket, &val, &segIdx, &slotIdx); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - // double val = *(double *)d; - double val = GET_DOUBLE_VAL(d); - (pBucket->HashFunc)(pBucket, &val, &segIdx, &slotIdx); - break; - } - case TSDB_DATA_TYPE_FLOAT: { - // double val = *(float *)d; - double val = GET_FLOAT_VAL(d); - (pBucket->HashFunc)(pBucket, &val, &segIdx, &slotIdx); - break; - } - } + for (int32_t i = 0; i < size; ++i) { + char *d = (char *) data + i * bytes; - tMemBucketSegment *pSeg = &pBucket->pSegs[segIdx]; - if (pSeg->pBoundingEntries == NULL) { - pSeg->pBoundingEntries = (MinMaxEntry *)malloc(sizeof(MinMaxEntry) * pBucket->nSlotsOfSeg); - resetBoundingBox(pSeg, pBucket->dataType); - } + int32_t slotIdx = (pBucket->hashFunc)(pBucket, d); + assert(slotIdx >= 0); - if (pSeg->pBuffer == NULL) { - pSeg->pBuffer = (tExtMemBuffer **)calloc(pBucket->nSlotsOfSeg, sizeof(void *)); - } - - if (pSeg->pBuffer[slotIdx] == NULL) { - pSeg->pBuffer[slotIdx] = createExtMemBuffer(pBucket->numOfTotalPages * pBucket->pageSize, pBucket->nElemSize, - pBucket->pageSize, pBucket->pOrderDesc->pColumnModel); - pSeg->pBuffer[slotIdx]->flushModel = SINGLE_APPEND_MODEL; - pBucket->pOrderDesc->pColumnModel->capacity = pSeg->pBuffer[slotIdx]->numOfElemsPerPage; - } - - tMemBucketUpdateBoundingBox(&pSeg->pBoundingEntries[slotIdx], d, pBucket->dataType); + tMemBucketSlot *pSlot = &pBucket->pSlots[slotIdx]; + tMemBucketUpdateBoundingBox(&pSlot->range, d, pBucket->type); // ensure available memory pages to allocate - int16_t cseg = 0, cslot = 0; - if (pBucket->numOfAvailPages == 0) { - uDebug("MemBucket:%p,max avail size:%d, no avail memory pages,", pBucket, pBucket->numOfTotalPages); + int32_t groupId = getGroupId(pBucket->numOfSlots, slotIdx, pBucket->times); + int32_t pageId = -1; - tBucketGetMaxMemSlot(pBucket, &cseg, &cslot); - if (cseg == -1 || cslot == -1) { - uError("MemBucket:%p,failed to find appropriated avail buffer", pBucket); - return; + if (pSlot->info.data == NULL || pSlot->info.data->num >= pBucket->elemPerPage) { + if (pSlot->info.data != NULL) { + assert(pSlot->info.data->num >= pBucket->elemPerPage && pSlot->info.size > 0); + + // keep the pointer in memory + releaseResBufPage(pBucket->pBuffer, pSlot->info.data); + pSlot->info.data = NULL; } - if (cseg != segIdx || cslot != slotIdx) { - pBucket->numOfAvailPages += pBucket->pSegs[cseg].pBuffer[cslot]->numOfInMemPages; - - int32_t avail = pBucket->pSegs[cseg].pBuffer[cslot]->numOfInMemPages; - UNUSED(avail); - tExtMemBufferFlush(pBucket->pSegs[cseg].pBuffer[cslot]); - - uDebug("MemBucket:%p,seg:%d,slot:%d flushed to disk,new avail pages:%d", pBucket, cseg, cslot, - pBucket->numOfAvailPages); - } else { - uDebug("MemBucket:%p,failed to choose slot to flush to disk seg:%d,slot:%d", pBucket, cseg, cslot); - } + pSlot->info.data = getNewDataBuf(pBucket->pBuffer, groupId, &pageId); + pSlot->info.pageId = pageId; } - int16_t consumedPgs = pSeg->pBuffer[slotIdx]->numOfInMemPages; - int16_t newPgs = tExtMemBufferPut(pSeg->pBuffer[slotIdx], d, 1); - /* - * trigger 1. page re-allocation, to reduce the available pages - * 2. page flushout, to increase the available pages - */ - pBucket->numOfAvailPages += (consumedPgs - newPgs); + memcpy(pSlot->info.data->data + pSlot->info.data->num * pBucket->bytes, d, pBucket->bytes); + + pSlot->info.data->num += 1; + pSlot->info.size += 1; } } -void releaseBucket(tMemBucket *pMemBucket, int32_t segIdx, int32_t slotIdx) { - if (segIdx < 0 || segIdx > pMemBucket->numOfSegs || slotIdx < 0) { - return; - } - - tMemBucketSegment *pSeg = &pMemBucket->pSegs[segIdx]; - if (slotIdx < 0 || slotIdx >= pSeg->numOfSlots || pSeg->pBuffer[slotIdx] == NULL) { - return; - } - - pSeg->pBuffer[slotIdx] = destoryExtMemBuffer(pSeg->pBuffer[slotIdx]); -} - //////////////////////////////////////////////////////////////////////////////////////////// static void findMaxMinValue(tMemBucket *pMemBucket, double *maxVal, double *minVal) { *minVal = DBL_MAX; *maxVal = -DBL_MAX; - for (int32_t i = 0; i < pMemBucket->numOfSegs; ++i) { - tMemBucketSegment *pSeg = &pMemBucket->pSegs[i]; - if (pSeg->pBuffer == NULL) { + for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) { + tMemBucketSlot *pSlot = &pMemBucket->pSlots[i]; + if (pSlot->info.size == 0) { continue; } - switch (pMemBucket->dataType) { + + switch (pMemBucket->type) { case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_TINYINT: { - for (int32_t j = 0; j < pSeg->numOfSlots; ++j) { - double minv = pSeg->pBoundingEntries[j].iMinVal; - double maxv = pSeg->pBoundingEntries[j].iMaxVal; + double minv = pSlot->range.iMinVal; + double maxv = pSlot->range.iMaxVal; - if (*minVal > minv) { - *minVal = minv; - } - if (*maxVal < maxv) { - *maxVal = maxv; - } + if (*minVal > minv) { + *minVal = minv; + } + if (*maxVal < maxv) { + *maxVal = maxv; } break; } case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_FLOAT: { - for (int32_t j = 0; j < pSeg->numOfSlots; ++j) { - double minv = pSeg->pBoundingEntries[j].dMinVal; - double maxv = pSeg->pBoundingEntries[j].dMaxVal; + double minv = pSlot->range.dMinVal; + double maxv = pSlot->range.dMaxVal; - if (*minVal > minv) { - *minVal = minv; - } - if (*maxVal < maxv) { - *maxVal = maxv; - } + if (*minVal > minv) { + *minVal = minv; + } + if (*maxVal < maxv) { + *maxVal = maxv; } break; } case TSDB_DATA_TYPE_BIGINT: { - for (int32_t j = 0; j < pSeg->numOfSlots; ++j) { - double minv = (double)pSeg->pBoundingEntries[j].i64MinVal; - double maxv = (double)pSeg->pBoundingEntries[j].i64MaxVal; + double minv = (double)pSlot->range.i64MinVal; + double maxv = (double)pSlot->range.i64MaxVal; - if (*minVal > minv) { - *minVal = minv; - } - if (*maxVal < maxv) { - *maxVal = maxv; - } + if (*minVal > minv) { + *minVal = minv; + } + if (*maxVal < maxv) { + *maxVal = maxv; } break; } @@ -650,20 +484,6 @@ static void findMaxMinValue(tMemBucket *pMemBucket, double *maxVal, double *minV } } -static MinMaxEntry getMinMaxEntryOfNearestSlotInNextSegment(tMemBucket *pMemBucket, int32_t segIdx) { - int32_t i = segIdx + 1; - while (i < pMemBucket->numOfSegs && pMemBucket->pSegs[i].numOfSlots == 0) ++i; - - tMemBucketSegment *pSeg = &pMemBucket->pSegs[i]; - assert(pMemBucket->numOfSegs > i && pMemBucket->pSegs[i].pBuffer != NULL); - - i = 0; - while (i < pMemBucket->nSlotsOfSeg && pSeg->pBuffer[i] == NULL) ++i; - - assert(i < pMemBucket->nSlotsOfSeg); - return pSeg->pBoundingEntries[i]; -} - /* * * now, we need to find the minimum value of the next slot for @@ -671,262 +491,198 @@ static MinMaxEntry getMinMaxEntryOfNearestSlotInNextSegment(tMemBucket *pMemBuck * j is the last slot of current segment, we need to get the first * slot of the next segment. */ -static MinMaxEntry getMinMaxEntryOfNextSlotWithData(tMemBucket *pMemBucket, int32_t segIdx, int32_t slotIdx) { - tMemBucketSegment *pSeg = &pMemBucket->pSegs[segIdx]; - - MinMaxEntry next; - if (slotIdx == pSeg->numOfSlots - 1) { // find next segment with data - return getMinMaxEntryOfNearestSlotInNextSegment(pMemBucket, segIdx); - } else { +static MinMaxEntry getMinMaxEntryOfNextSlotWithData(tMemBucket *pMemBucket, int32_t slotIdx) { int32_t j = slotIdx + 1; - for (; j < pMemBucket->nSlotsOfSeg && pMemBucket->pSegs[segIdx].pBuffer[j] == 0; ++j) { + while (j < pMemBucket->numOfSlots && (pMemBucket->pSlots[j].info.size == 0)) { + ++j; + } + + assert(j < pMemBucket->numOfSlots); + return pMemBucket->pSlots[j].range; +} + +static bool isIdenticalData(tMemBucket *pMemBucket, int32_t index); +char *getFirstElemOfMemBuffer(tMemBucketSlot *pSeg, int32_t slotIdx, tFilePage *pPage); + +static double getIdenticalDataVal(tMemBucket* pMemBucket, int32_t slotIndex) { + assert(isIdenticalData(pMemBucket, slotIndex)); + + tMemBucketSlot *pSlot = &pMemBucket->pSlots[slotIndex]; + + double finalResult = 0.0; + switch (pMemBucket->type) { + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_INT: { + finalResult = pSlot->range.iMinVal; + break; + } + + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + finalResult = pSlot->range.dMinVal; + break; }; - if (j == pMemBucket->nSlotsOfSeg) { // current slot has no available - // slot,try next segment - return getMinMaxEntryOfNearestSlotInNextSegment(pMemBucket, segIdx); - } else { - next = pSeg->pBoundingEntries[slotIdx + 1]; - assert(pSeg->pBuffer[slotIdx + 1] != NULL); + case TSDB_DATA_TYPE_BIGINT: { + finalResult = (double)pSlot->range.i64MinVal; + break; } } - return next; + return finalResult; } -bool isIdenticalData(tMemBucket *pMemBucket, int32_t segIdx, int32_t slotIdx); -char *getFirstElemOfMemBuffer(tMemBucketSegment *pSeg, int32_t slotIdx, tFilePage *pPage); - double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) { int32_t num = 0; - for (int32_t i = 0; i < pMemBucket->numOfSegs; ++i) { - tMemBucketSegment *pSeg = &pMemBucket->pSegs[i]; - for (int32_t j = 0; j < pSeg->numOfSlots; ++j) { - if (pSeg->pBuffer == NULL || pSeg->pBuffer[j] == NULL) { - continue; - } - // required value in current slot - if (num < (count + 1) && num + pSeg->pBuffer[j]->numOfTotalElems >= (count + 1)) { - if (pSeg->pBuffer[j]->numOfTotalElems + num == (count + 1)) { - /* - * now, we need to find the minimum value of the next slot for interpolating the percentile value - * j is the last slot of current segment, we need to get the first slot of the next segment. - * - */ - MinMaxEntry next = getMinMaxEntryOfNextSlotWithData(pMemBucket, i, j); + for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) { + tMemBucketSlot *pSlot = &pMemBucket->pSlots[i]; + if (pSlot->info.size == 0) { + continue; + } - double maxOfThisSlot = 0; - double minOfNextSlot = 0; - switch (pMemBucket->dataType) { - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_TINYINT: { - maxOfThisSlot = pSeg->pBoundingEntries[j].iMaxVal; - minOfNextSlot = next.iMinVal; - break; - }; - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: { - maxOfThisSlot = pSeg->pBoundingEntries[j].dMaxVal; - minOfNextSlot = next.dMinVal; - break; - }; - case TSDB_DATA_TYPE_BIGINT: { - maxOfThisSlot = (double)pSeg->pBoundingEntries[j].i64MaxVal; - minOfNextSlot = (double)next.i64MinVal; - break; - } + // required value in current slot + if (num < (count + 1) && num + pSlot->info.size >= (count + 1)) { + if (pSlot->info.size + num == (count + 1)) { + /* + * now, we need to find the minimum value of the next slot for interpolating the percentile value + * j is the last slot of current segment, we need to get the first slot of the next segment. + */ + MinMaxEntry next = getMinMaxEntryOfNextSlotWithData(pMemBucket, i); + + double maxOfThisSlot = 0; + double minOfNextSlot = 0; + switch (pMemBucket->type) { + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_TINYINT: { + maxOfThisSlot = pSlot->range.iMaxVal; + minOfNextSlot = next.iMinVal; + break; }; - - assert(minOfNextSlot > maxOfThisSlot); - - double val = (1 - fraction) * maxOfThisSlot + fraction * minOfNextSlot; - return val; - } - if (pSeg->pBuffer[j]->numOfTotalElems <= pMemBucket->maxElemsCapacity) { - // data in buffer and file are merged together to be processed. - tFilePage *buffer = loadIntoBucketFromDisk(pMemBucket, i, j, pMemBucket->pOrderDesc); - int32_t currentIdx = count - num; - - char * thisVal = buffer->data + pMemBucket->nElemSize * currentIdx; - char * nextVal = thisVal + pMemBucket->nElemSize; - double td = 1.0, nd = 1.0; - switch (pMemBucket->dataType) { - case TSDB_DATA_TYPE_SMALLINT: { - td = *(int16_t *)thisVal; - nd = *(int16_t *)nextVal; - break; - } - case TSDB_DATA_TYPE_TINYINT: { - td = *(int8_t *)thisVal; - nd = *(int8_t *)nextVal; - break; - } - case TSDB_DATA_TYPE_INT: { - td = *(int32_t *)thisVal; - nd = *(int32_t *)nextVal; - break; - }; - case TSDB_DATA_TYPE_FLOAT: { - // td = *(float *)thisVal; - // nd = *(float *)nextVal; - td = GET_FLOAT_VAL(thisVal); - nd = GET_FLOAT_VAL(nextVal); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - // td = *(double *)thisVal; - td = GET_DOUBLE_VAL(thisVal); - // nd = *(double *)nextVal; - nd = GET_DOUBLE_VAL(nextVal); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - td = (double)*(int64_t *)thisVal; - nd = (double)*(int64_t *)nextVal; - break; - } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + maxOfThisSlot = pSlot->range.dMaxVal; + minOfNextSlot = next.dMinVal; + break; + }; + case TSDB_DATA_TYPE_BIGINT: { + maxOfThisSlot = (double)pSlot->range.i64MaxVal; + minOfNextSlot = (double)next.i64MinVal; + break; } - double val = (1 - fraction) * td + fraction * nd; - taosTFree(buffer); + }; - return val; - } else { // incur a second round bucket split - if (isIdenticalData(pMemBucket, i, j)) { - tExtMemBuffer *pMemBuffer = pSeg->pBuffer[j]; + assert(minOfNextSlot > maxOfThisSlot); - tFilePage *pPage = (tFilePage *)malloc(pMemBuffer->pageSize); - - char *thisVal = getFirstElemOfMemBuffer(pSeg, j, pPage); - - double finalResult = 0.0; - - switch (pMemBucket->dataType) { - case TSDB_DATA_TYPE_SMALLINT: { - finalResult = *(int16_t *)thisVal; - break; - } - case TSDB_DATA_TYPE_TINYINT: { - finalResult = *(int8_t *)thisVal; - break; - } - case TSDB_DATA_TYPE_INT: { - finalResult = *(int32_t *)thisVal; - break; - }; - case TSDB_DATA_TYPE_FLOAT: { - // finalResult = *(float *)thisVal; - finalResult = GET_FLOAT_VAL(thisVal); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - // finalResult = *(double *)thisVal; - finalResult = GET_DOUBLE_VAL(thisVal); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - finalResult = (double)(*(int64_t *)thisVal); - break; - } - } - - free(pPage); - return finalResult; - } - - uDebug("MemBucket:%p,start second round bucketing", pMemBucket); - - if (pSeg->pBuffer[j]->numOfElemsInBuffer != 0) { - uDebug("MemBucket:%p,flush %d pages to disk, clear status", pMemBucket, pSeg->pBuffer[j]->numOfInMemPages); - - pMemBucket->numOfAvailPages += pSeg->pBuffer[j]->numOfInMemPages; - tExtMemBufferFlush(pSeg->pBuffer[j]); - } - - tExtMemBuffer *pMemBuffer = pSeg->pBuffer[j]; - pSeg->pBuffer[j] = NULL; - - // release all - for (int32_t tt = 0; tt < pMemBucket->numOfSegs; ++tt) { - tMemBucketSegment *pSeg = &pMemBucket->pSegs[tt]; - for (int32_t ttx = 0; ttx < pSeg->numOfSlots; ++ttx) { - if (pSeg->pBuffer && pSeg->pBuffer[ttx]) { - pSeg->pBuffer[ttx] = destoryExtMemBuffer(pSeg->pBuffer[ttx]); - } - } - } - - pMemBucket->nRange.i64MaxVal = pSeg->pBoundingEntries->i64MaxVal; - pMemBucket->nRange.i64MinVal = pSeg->pBoundingEntries->i64MinVal; - pMemBucket->numOfElems = 0; - - for (int32_t tt = 0; tt < pMemBucket->numOfSegs; ++tt) { - tMemBucketSegment *pSeg = &pMemBucket->pSegs[tt]; - for (int32_t ttx = 0; ttx < pSeg->numOfSlots; ++ttx) { - if (pSeg->pBoundingEntries) { - resetBoundingBox(pSeg, pMemBucket->dataType); - } - } - } - - tFilePage *pPage = (tFilePage *)malloc(pMemBuffer->pageSize); - - tFlushoutInfo *pFlushInfo = &pMemBuffer->fileMeta.flushoutData.pFlushoutInfo[0]; - assert(pFlushInfo->numOfPages == pMemBuffer->fileMeta.nFileSize); - - int32_t ret = fseek(pMemBuffer->file, pFlushInfo->startPageId * pMemBuffer->pageSize, SEEK_SET); - UNUSED(ret); - - for (uint32_t jx = 0; jx < pFlushInfo->numOfPages; ++jx) { - size_t sz = fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file); - if (sz != pMemBuffer->pageSize) { - uError("MemBucket:%p, read tmp file %s failed", pMemBucket, pMemBuffer->path); - } else { - tMemBucketPut(pMemBucket, pPage->data, (int32_t)pPage->num); - } - } - - fclose(pMemBuffer->file); - if (unlink(pMemBuffer->path) != 0) { - uError("MemBucket:%p, remove tmp file %s failed", pMemBucket, pMemBuffer->path); - } - taosTFree(pMemBuffer); - taosTFree(pPage); - - return getPercentileImpl(pMemBucket, count - num, fraction); - } - } else { - num += pSeg->pBuffer[j]->numOfTotalElems; + double val = (1 - fraction) * maxOfThisSlot + fraction * minOfNextSlot; + return val; } + + if (pSlot->info.size <= pMemBucket->maxCapacity) { + // data in buffer and file are merged together to be processed. + tFilePage *buffer = loadDataFromFilePage(pMemBucket, i); + int32_t currentIdx = count - num; + + char *thisVal = buffer->data + pMemBucket->bytes * currentIdx; + char *nextVal = thisVal + pMemBucket->bytes; + + double td = 1.0, nd = 1.0; + switch (pMemBucket->type) { + case TSDB_DATA_TYPE_SMALLINT: { + td = *(int16_t *)thisVal; + nd = *(int16_t *)nextVal; + break; + } + case TSDB_DATA_TYPE_TINYINT: { + td = *(int8_t *)thisVal; + nd = *(int8_t *)nextVal; + break; + } + case TSDB_DATA_TYPE_INT: { + td = *(int32_t *)thisVal; + nd = *(int32_t *)nextVal; + break; + }; + case TSDB_DATA_TYPE_FLOAT: { + td = GET_FLOAT_VAL(thisVal); + nd = GET_FLOAT_VAL(nextVal); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + td = GET_DOUBLE_VAL(thisVal); + nd = GET_DOUBLE_VAL(nextVal); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + td = (double)*(int64_t *)thisVal; + nd = (double)*(int64_t *)nextVal; + break; + } + } + + double val = (1 - fraction) * td + fraction * nd; + taosTFree(buffer); + + return val; + } else { // incur a second round bucket split + if (isIdenticalData(pMemBucket, i)) { + return getIdenticalDataVal(pMemBucket, i); + } + + // try next round + pMemBucket->times += 1; + uDebug("MemBucket:%p, start next round data bucketing, time:%d", pMemBucket, pMemBucket->times); + + pMemBucket->range = pSlot->range; + pMemBucket->total = 0; + + resetSlotInfo(pMemBucket); + + int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times - 1); + SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId); + assert(list->size > 0); + + for (int32_t f = 0; f < list->size; ++f) { + SPageInfo *pgInfo = *(SPageInfo **)taosArrayGet(list, f); + tFilePage *pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId); + + tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num); + releaseResBufPageInfo(pMemBucket->pBuffer, pgInfo); + } + + return getPercentileImpl(pMemBucket, count - num, fraction); + } + } else { + num += pSlot->info.size; } } + return 0; } double getPercentile(tMemBucket *pMemBucket, double percent) { - if (pMemBucket->numOfElems == 0) { + if (pMemBucket->total == 0) { return 0.0; } - if (pMemBucket->numOfElems == 1) { // return the only element + // if only one elements exists, return it + if (pMemBucket->total == 1) { return findOnlyResult(pMemBucket); } percent = fabs(percent); - // validate the parameters + // find the min/max value, no need to scan all data in bucket if (fabs(percent - 100.0) < DBL_EPSILON || (percent < DBL_EPSILON)) { double minx = 0, maxx = 0; - /* - * find the min/max value, no need to scan all data in bucket - */ findMaxMinValue(pMemBucket, &maxx, &minx); return fabs(percent - 100) < DBL_EPSILON ? maxx : minx; } - double percentVal = (percent * (pMemBucket->numOfElems - 1)) / ((double)100.0); + double percentVal = (percent * (pMemBucket->total - 1)) / ((double)100.0); int32_t orderIdx = (int32_t)percentVal; // do put data by using buckets @@ -934,19 +690,18 @@ double getPercentile(tMemBucket *pMemBucket, double percent) { } /* - * check if data in one slot are all identical - * only need to compare with the bounding box + * check if data in one slot are all identical only need to compare with the bounding box */ -bool isIdenticalData(tMemBucket *pMemBucket, int32_t segIdx, int32_t slotIdx) { - tMemBucketSegment *pSeg = &pMemBucket->pSegs[segIdx]; +bool isIdenticalData(tMemBucket *pMemBucket, int32_t index) { + tMemBucketSlot *pSeg = &pMemBucket->pSlots[index]; - if (pMemBucket->dataType == TSDB_DATA_TYPE_INT || pMemBucket->dataType == TSDB_DATA_TYPE_BIGINT || - pMemBucket->dataType == TSDB_DATA_TYPE_SMALLINT || pMemBucket->dataType == TSDB_DATA_TYPE_TINYINT) { - return pSeg->pBoundingEntries[slotIdx].i64MinVal == pSeg->pBoundingEntries[slotIdx].i64MaxVal; + if (pMemBucket->type == TSDB_DATA_TYPE_INT || pMemBucket->type == TSDB_DATA_TYPE_BIGINT || + pMemBucket->type == TSDB_DATA_TYPE_SMALLINT || pMemBucket->type == TSDB_DATA_TYPE_TINYINT) { + return pSeg->range.i64MinVal == pSeg->range.i64MaxVal; } - if (pMemBucket->dataType == TSDB_DATA_TYPE_FLOAT || pMemBucket->dataType == TSDB_DATA_TYPE_DOUBLE) { - return fabs(pSeg->pBoundingEntries[slotIdx].dMaxVal - pSeg->pBoundingEntries[slotIdx].dMinVal) < DBL_EPSILON; + if (pMemBucket->type == TSDB_DATA_TYPE_FLOAT || pMemBucket->type == TSDB_DATA_TYPE_DOUBLE) { + return fabs(pSeg->range.dMaxVal - pSeg->range.dMinVal) < DBL_EPSILON; } return false; @@ -956,24 +711,24 @@ bool isIdenticalData(tMemBucket *pMemBucket, int32_t segIdx, int32_t slotIdx) { * get the first element of one slot into memory. * if no data of current slot in memory, load it from disk */ -char *getFirstElemOfMemBuffer(tMemBucketSegment *pSeg, int32_t slotIdx, tFilePage *pPage) { - tExtMemBuffer *pMemBuffer = pSeg->pBuffer[slotIdx]; - char * thisVal = NULL; +char *getFirstElemOfMemBuffer(tMemBucketSlot *pSeg, int32_t slotIdx, tFilePage *pPage) { +// STSBuf *pMemBuffer = pSeg->pBuffer[slotIdx]; + char *thisVal = NULL; - if (pSeg->pBuffer[slotIdx]->numOfElemsInBuffer != 0) { - thisVal = pSeg->pBuffer[slotIdx]->pHead->item.data; - } else { - /* - * no data in memory, load one page into memory - */ - tFlushoutInfo *pFlushInfo = &pMemBuffer->fileMeta.flushoutData.pFlushoutInfo[0]; - assert(pFlushInfo->numOfPages == pMemBuffer->fileMeta.nFileSize); - int32_t ret; - ret = fseek(pMemBuffer->file, pFlushInfo->startPageId * pMemBuffer->pageSize, SEEK_SET); - UNUSED(ret); - size_t sz = fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file); - UNUSED(sz); - thisVal = pPage->data; - } +// if (pSeg->pBuffer[slotIdx]->numOfTotal != 0) { +//// thisVal = pSeg->pBuffer[slotIdx]->pHead->item.data; +// } else { +// /* +// * no data in memory, load one page into memory +// */ +// tFlushoutInfo *pFlushInfo = &pMemBuffer->fileMeta.flushoutData.pFlushoutInfo[0]; +// assert(pFlushInfo->numOfPages == pMemBuffer->fileMeta.nFileSize); +// int32_t ret; +// ret = fseek(pMemBuffer->file, pFlushInfo->startPageId * pMemBuffer->pageSize, SEEK_SET); +// UNUSED(ret); +// size_t sz = fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file); +// UNUSED(sz); +// thisVal = pPage->data; +// } return thisVal; } diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 4c6d75ec14..84f22918ec 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1130,8 +1130,15 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32 // Decode the data if (comp) { // // Need to decompress - pDataCol->len = (*(tDataTypeDesc[pDataCol->type].decompFunc))( - content, len - sizeof(TSCKSUM), numOfRows, pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize); + int tlen = (*(tDataTypeDesc[pDataCol->type].decompFunc))(content, len - sizeof(TSCKSUM), numOfRows, pDataCol->pData, + pDataCol->spaceSize, comp, buffer, bufferSize); + if (tlen <= 0) { + tsdbError("Failed to decompress column, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d", + len, comp, numOfRows, maxPoints, bufferSize); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } + pDataCol->len = tlen; if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { dataColSetOffset(pDataCol, numOfRows); } diff --git a/src/util/inc/tnettest.h b/src/util/inc/tnettest.h new file mode 100644 index 0000000000..3fe1dfa920 --- /dev/null +++ b/src/util/inc/tnettest.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TNETTEST_H +#define TDENGINE_TNETTEST_H + +#ifdef __cplusplus +extern "C" { +#endif + +void taosNetTest(const char* host, uint16_t port, uint16_t endPort, int pktLen, const char* netTestRole); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TNETTEST_H diff --git a/src/util/inc/tscompression.h b/src/util/inc/tscompression.h index bd1ccf3ca5..37d1e7b590 100644 --- a/src/util/inc/tscompression.h +++ b/src/util/inc/tscompression.h @@ -65,7 +65,7 @@ static FORCE_INLINE int tsDecompressTinyint(const char *const input, int compres if (algorithm == ONE_STAGE_COMP) { return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); } else if (algorithm == TWO_STAGE_COMP) { - tsDecompressStringImp(input, compressedSize, buffer, bufferSize); + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT); } else { assert(0); @@ -91,7 +91,7 @@ static FORCE_INLINE int tsDecompressSmallint(const char *const input, int compre if (algorithm == ONE_STAGE_COMP) { return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); } else if (algorithm == TWO_STAGE_COMP) { - tsDecompressStringImp(input, compressedSize, buffer, bufferSize); + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT); } else { assert(0); @@ -117,7 +117,7 @@ static FORCE_INLINE int tsDecompressInt(const char *const input, int compressedS if (algorithm == ONE_STAGE_COMP) { return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); } else if (algorithm == TWO_STAGE_COMP) { - tsDecompressStringImp(input, compressedSize, buffer, bufferSize); + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT); } else { assert(0); @@ -143,7 +143,7 @@ static FORCE_INLINE int tsDecompressBigint(const char *const input, int compress if (algorithm == ONE_STAGE_COMP) { return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); } else if (algorithm == TWO_STAGE_COMP) { - tsDecompressStringImp(input, compressedSize, buffer, bufferSize); + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT); } else { assert(0); @@ -169,7 +169,7 @@ static FORCE_INLINE int tsDecompressBool(const char *const input, int compressed if (algorithm == ONE_STAGE_COMP) { return tsDecompressBoolImp(input, nelements, output); } else if (algorithm == TWO_STAGE_COMP) { - tsDecompressStringImp(input, compressedSize, buffer, bufferSize); + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; return tsDecompressBoolImp(buffer, nelements, output); } else { assert(0); @@ -205,7 +205,7 @@ static FORCE_INLINE int tsDecompressFloat(const char *const input, int compresse if (algorithm == ONE_STAGE_COMP) { return tsDecompressFloatImp(input, nelements, output); } else if (algorithm == TWO_STAGE_COMP) { - tsDecompressStringImp(input, compressedSize, buffer, bufferSize); + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; return tsDecompressFloatImp(buffer, nelements, output); } else { assert(0); @@ -231,7 +231,7 @@ static FORCE_INLINE int tsDecompressDouble(const char *const input, int compress if (algorithm == ONE_STAGE_COMP) { return tsDecompressDoubleImp(input, nelements, output); } else if (algorithm == TWO_STAGE_COMP) { - tsDecompressStringImp(input, compressedSize, buffer, bufferSize); + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; return tsDecompressDoubleImp(buffer, nelements, output); } else { assert(0); @@ -257,7 +257,7 @@ static FORCE_INLINE int tsDecompressTimestamp(const char *const input, int compr if (algorithm == ONE_STAGE_COMP) { return tsDecompressTimestampImp(input, nelements, output); } else if (algorithm == TWO_STAGE_COMP) { - tsDecompressStringImp(input, compressedSize, buffer, bufferSize); + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; return tsDecompressTimestampImp(buffer, nelements, output); } else { assert(0); diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 9634175db7..cc96f83f44 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -327,7 +327,6 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe // no data, return directly if (pe->num == 0) { - assert(pe->next == NULL); __rd_unlock(&pHashObj->lock, pHashObj->type); return -1; } diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index dfa982b848..3b14254fff 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -266,7 +266,12 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen } SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGetCB(pCacheObj->pHashTable, key, keyLen, incRefFn); + if (ptNode != NULL) { + assert ((*ptNode) != NULL && (int64_t) ((*ptNode)->data) != 0x40); + } + void* pData = (ptNode != NULL)? (*ptNode)->data:NULL; + assert((int64_t)pData != 0x40); if (pData != NULL) { atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); @@ -349,7 +354,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { char* d = pNode->data; int32_t ref = T_REF_VAL_GET(pNode); - uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, key, d, ref - 1); + uDebug("cache:%s, key:%p, %p is released, refcnt:%d, intrash:%d", pCacheObj->name, key, d, ref - 1, inTrashCan); /* * If it is not referenced by other users, remove it immediately. Otherwise move this node to trashcan wait for all users @@ -373,18 +378,24 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { } else { // NOTE: remove it from hash in the first place, otherwise, the pNode may have been released by other thread // when reaches here. - int32_t ret = taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); + SCacheDataNode* p = NULL; + int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, pNode->key, pNode->keySize, &p, sizeof(void*)); ref = T_REF_DEC(pNode); // successfully remove from hash table, if failed, this node must have been move to trash already, do nothing. // note that the remove operation can be executed only once. if (ret == 0) { + if (p != pNode) { + uDebug("cache:%s, key:%p, successfully removed a new entry:%p, refcnt:%d, prev entry:%p has been removed by others already", pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data); + assert(p->pTNodeHeader == NULL); + taosAddToTrash(pCacheObj, p); + } else { + + uDebug("cache:%s, key:%p, %p successfully removed from hash table, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, ref); if (ref > 0) { assert(pNode->pTNodeHeader == NULL); - __cache_wr_lock(pCacheObj); taosAddToTrash(pCacheObj, pNode); - __cache_unlock(pCacheObj); } else { // ref == 0 atomic_sub_fetch_64(&pCacheObj->totalSize, pNode->size); @@ -398,6 +409,9 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { free(pNode); } + } + } else { + uDebug("cache:%s, key:%p, %p has been removed from hash table by other thread already, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, ref); } } @@ -485,18 +499,19 @@ void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { STrashElem *pElem = calloc(1, sizeof(STrashElem)); pElem->pData = pNode; + pElem->prev = NULL; + pNode->inTrashCan = true; + pNode->pTNodeHeader = pElem; + __cache_wr_lock(pCacheObj); pElem->next = pCacheObj->pTrash; if (pCacheObj->pTrash) { pCacheObj->pTrash->prev = pElem; } - pElem->prev = NULL; pCacheObj->pTrash = pElem; - - pNode->inTrashCan = true; - pNode->pTNodeHeader = pElem; pCacheObj->numOfElemsInTrash++; + __cache_unlock(pCacheObj); uDebug("%s key:%p, %p move to trash, numOfElem in trash:%d", pCacheObj->name, pNode->key, pNode->data, pCacheObj->numOfElemsInTrash); diff --git a/src/util/src/tcompression.c b/src/util/src/tcompression.c index 8c5828d32d..1a5d28625f 100644 --- a/src/util/src/tcompression.c +++ b/src/util/src/tcompression.c @@ -47,10 +47,11 @@ * */ -#include "os.h" #include "lz4.h" -#include "tscompression.h" +#include "os.h" #include "taosdef.h" +#include "tscompression.h" +#include "tulog.h" static const int TEST_NUMBER = 1; #define is_bigendian() ((*(char *)&TEST_NUMBER) == 0) @@ -88,7 +89,7 @@ int tsCompressINTImp(const char *const input, const int nelements, char *const o word_length = CHAR_BYTES; break; default: - perror("Wrong integer types.\n"); + uError("Invalid compress integer type:%d", type); return -1; } @@ -209,7 +210,7 @@ int tsDecompressINTImp(const char *const input, const int nelements, char *const word_length = CHAR_BYTES; break; default: - perror("Wrong integer types.\n"); + uError("Invalid decompress integer type:%d", type); return -1; } @@ -307,7 +308,7 @@ int tsCompressBoolImp(const char *const input, const int nelements, char *const /* t = (~((( uint8_t)1) << (7-i%BITS_PER_BYTE))); */ output[pos] |= t; } else { - perror("Wrong bool value.\n"); + uError("Invalid compress bool value:%d", output[pos]); return -1; } } @@ -363,7 +364,7 @@ int tsCompressBoolRLEImp(const char *const input, const int nelements, char *con } else if (num == 0) { output[_pos++] = (counter << 1) | INT8MASK(0); } else { - perror("Wrong bool value!\n"); + uError("Invalid compress bool value:%d", output[_pos]); return -1; } } @@ -413,9 +414,7 @@ int tsDecompressStringImp(const char *const input, int compressedSize, char *con /* It is compressed by LZ4 algorithm */ const int decompressed_size = LZ4_decompress_safe(input + 1, output, compressedSize - 1, outputSize); if (decompressed_size < 0) { - char msg[128] = {0}; - sprintf(msg, "decomp_size:%d, Error decompress in LZ4 algorithm!\n", decompressed_size); - perror(msg); + uError("Failed to decompress string with LZ4 algorithm, decompressed size:%d", decompressed_size); return -1; } @@ -425,7 +424,7 @@ int tsDecompressStringImp(const char *const input, int compressedSize, char *con memcpy(output, input + 1, compressedSize - 1); return compressedSize - 1; } else { - perror("Wrong compressed string indicator!\n"); + uError("Invalid decompress string indicator:%d", input[0]); return -1; } } diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c new file mode 100644 index 0000000000..5a1430baed --- /dev/null +++ b/src/util/src/tnettest.c @@ -0,0 +1,423 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "taosdef.h" +#include "taoserror.h" +#include "tulog.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tsocket.h" + +#define MAX_PKG_LEN (64*1000) +#define BUFFER_SIZE (MAX_PKG_LEN + 1024) + +typedef struct { + uint32_t hostIp; + uint16_t port; + uint16_t pktLen; +} info_s; + +static char serverFqdn[TSDB_FQDN_LEN]; +static uint16_t g_startPort = 0; +static uint16_t g_endPort = 6042; + +static void *bindUdpPort(void *sarg) { + info_s *pinfo = (info_s *)sarg; + int port = pinfo->port; + SOCKET serverSocket; + + struct sockaddr_in server_addr; + struct sockaddr_in clientAddr; + char buffer[BUFFER_SIZE]; + int iDataNum; + + if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + perror("socket"); + return NULL; + } + + bzero(&server_addr, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + server_addr.sin_addr.s_addr = htonl(INADDR_ANY); + + if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { + perror("connect"); + return NULL; + } + + socklen_t sin_size; + + while (1) { + memset(buffer, 0, BUFFER_SIZE); + + sin_size = sizeof(*(struct sockaddr *)&server_addr); + + iDataNum = recvfrom(serverSocket, buffer, BUFFER_SIZE, 0, (struct sockaddr *)&clientAddr, &sin_size); + + if (iDataNum < 0) { + perror("recvfrom null"); + continue; + } + if (iDataNum > 0) { + printf("recv Client: %s pkg from UDP port: %d, pkg len: %d\n", inet_ntoa(clientAddr.sin_addr), port, iDataNum); + //printf("Read msg from udp:%s ... %s\n", buffer, buffer+iDataNum-16); + + sendto(serverSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int)sin_size); + } + } + + taosCloseSocket(serverSocket); + return NULL; +} + +static void *bindTcpPort(void *sarg) { + info_s *pinfo = (info_s *)sarg; + int port = pinfo->port; + SOCKET serverSocket; + + struct sockaddr_in server_addr; + struct sockaddr_in clientAddr; + int addr_len = sizeof(clientAddr); + SOCKET client; + char buffer[BUFFER_SIZE]; + int iDataNum = 0; + + if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { + printf("socket() fail: %s", strerror(errno)); + return NULL; + } + + bzero(&server_addr, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + server_addr.sin_addr.s_addr = htonl(INADDR_ANY); + + if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { + printf("port:%d bind() fail: %s", port, strerror(errno)); + return NULL; + } + + if (listen(serverSocket, 5) < 0) { + printf("listen() fail: %s", strerror(errno)); + return NULL; + } + + //printf("Bind port: %d success\n", port); + while (1) { + client = accept(serverSocket, (struct sockaddr *)&clientAddr, (socklen_t *)&addr_len); + if (client < 0) { + printf("accept() fail: %s", strerror(errno)); + continue; + } + + iDataNum = 0; + memset(buffer, 0, BUFFER_SIZE); + int nleft, nread; + char *ptr = buffer; + nleft = pinfo->pktLen; + while (nleft > 0) { + nread = recv(client, ptr, BUFFER_SIZE, 0); + + if (nread == 0) { + break; + } else if (nread < 0) { + if (errno == EINTR) { + continue; + } else { + printf("recv Client: %s pkg from TCP port: %d fail:%s.\n", inet_ntoa(clientAddr.sin_addr), port, strerror(errno)); + taosCloseSocket(serverSocket); + return NULL; + } + } else { + nleft -= nread; + ptr += nread; + iDataNum += nread; + } + } + + printf("recv Client: %s pkg from TCP port: %d, pkg len: %d\n", inet_ntoa(clientAddr.sin_addr), port, iDataNum); + if (iDataNum > 0) { + send(client, buffer, iDataNum, 0); + } + } + + taosCloseSocket(serverSocket); + return NULL; +} + +static int checkTcpPort(info_s *info) { + struct sockaddr_in serverAddr; + SOCKET clientSocket; + char sendbuf[BUFFER_SIZE]; + char recvbuf[BUFFER_SIZE]; + int iDataNum = 0; + if ((clientSocket = socket(AF_INET, SOCK_STREAM, 0)) < 0) { + printf("socket() fail: %s\n", strerror(errno)); + return -1; + } + + // set send and recv overtime + struct timeval timeout; + timeout.tv_sec = 2; //s + timeout.tv_usec = 0; //us + if (setsockopt(clientSocket, SOL_SOCKET,SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { + perror("setsockopt send timer failed:"); + } + if (setsockopt(clientSocket, SOL_SOCKET,SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { + perror("setsockopt recv timer failed:"); + } + + serverAddr.sin_family = AF_INET; + serverAddr.sin_port = htons(info->port); + + serverAddr.sin_addr.s_addr = info->hostIp; + + //printf("=================================\n"); + if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { + printf("connect() fail: %s\t", strerror(errno)); + return -1; + } + //printf("Connect to: %s:%d...success\n", host, port); + memset(sendbuf, 0, BUFFER_SIZE); + memset(recvbuf, 0, BUFFER_SIZE); + + struct in_addr ipStr; + memcpy(&ipStr, &info->hostIp, 4); + sprintf(sendbuf, "client send tcp pkg to %s:%d, content: 1122334455", inet_ntoa(ipStr), info->port); + sprintf(sendbuf + info->pktLen - 16, "1122334455667788"); + + send(clientSocket, sendbuf, info->pktLen, 0); + + memset(recvbuf, 0, BUFFER_SIZE); + int nleft, nread; + char *ptr = recvbuf; + nleft = info->pktLen; + while (nleft > 0) { + nread = recv(clientSocket, ptr, BUFFER_SIZE, 0);; + + if (nread == 0) { + break; + } else if (nread < 0) { + if (errno == EINTR) { + continue; + } else { + printf("recv ack pkg from TCP port: %d fail:%s.\n", info->port, strerror(errno)); + taosCloseSocket(clientSocket); + return -1; + } + } else { + nleft -= nread; + ptr += nread; + iDataNum += nread; + } + } + + if (iDataNum < info->pktLen) { + printf("recv ack pkg len: %d, less than req pkg len: %d from tcp port: %d\n", iDataNum, info->pktLen, info->port); + return -1; + } + //printf("Read ack pkg len:%d from tcp port: %d, buffer: %s %s\n", info->pktLen, port, recvbuf, recvbuf+iDataNum-8); + + taosCloseSocket(clientSocket); + return 0; +} + +static int checkUdpPort(info_s *info) { + struct sockaddr_in serverAddr; + SOCKET clientSocket; + char sendbuf[BUFFER_SIZE]; + char recvbuf[BUFFER_SIZE]; + int iDataNum = 0; + if ((clientSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + perror("socket"); + return -1; + } + + // set overtime + struct timeval timeout; + timeout.tv_sec = 2; //s + timeout.tv_usec = 0; //us + if (setsockopt(clientSocket, SOL_SOCKET,SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { + perror("setsockopt send timer failed:"); + } + if (setsockopt(clientSocket, SOL_SOCKET,SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { + perror("setsockopt recv timer failed:"); + } + + serverAddr.sin_family = AF_INET; + serverAddr.sin_port = htons(info->port); + serverAddr.sin_addr.s_addr = info->hostIp; + + memset(sendbuf, 0, BUFFER_SIZE); + memset(recvbuf, 0, BUFFER_SIZE); + + struct in_addr ipStr; + memcpy(&ipStr, &info->hostIp, 4); + sprintf(sendbuf, "client send udp pkg to %s:%d, content: 1122334455", inet_ntoa(ipStr), info->port); + sprintf(sendbuf + info->pktLen - 16, "1122334455667788"); + + socklen_t sin_size = sizeof(*(struct sockaddr *)&serverAddr); + + int code = sendto(clientSocket, sendbuf, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int)sin_size); + if (code < 0) { + perror("sendto"); + return -1; + } + + iDataNum = recvfrom(clientSocket, recvbuf, BUFFER_SIZE, 0, (struct sockaddr *)&serverAddr, &sin_size); + + if (iDataNum < info->pktLen) { + printf("Read ack pkg len: %d, less than req pkg len: %d from udp port: %d\t\t", iDataNum, info->pktLen, info->port); + return -1; + } + + //printf("Read ack pkg len:%d from udp port: %d, buffer: %s %s\n", info->pktLen, port, recvbuf, recvbuf+iDataNum-8); + taosCloseSocket(clientSocket); + return 0; +} + +static void checkPort(uint32_t hostIp, uint16_t startPort, uint16_t maxPort, uint16_t pktLen) { + int ret; + info_s info; + memset(&info, 0, sizeof(info_s)); + info.hostIp = hostIp; + info.pktLen = pktLen; + + for (uint16_t port = startPort; port <= maxPort; port++) { + //printf("test: %s:%d\n", info.host, port); + printf("\n"); + + info.port = port; + ret = checkTcpPort(&info); + if (ret != 0) { + printf("tcp port:%d test fail.\t\n", port); + } else { + printf("tcp port:%d test ok.\t\t", port); + } + + ret = checkUdpPort(&info); + if (ret != 0) { + printf("udp port:%d test fail.\t\n", port); + } else { + printf("udp port:%d test ok.\t\t", port); + } + } + + printf("\n"); + return ; +} + +static void taosNetTestClient(const char* serverFqdn, uint16_t startPort, uint16_t endPort, int pktLen) { + uint32_t serverIp = taosGetIpFromFqdn(serverFqdn); + if (serverIp == 0xFFFFFFFF) { + printf("Failed to resolve FQDN:%s", serverFqdn); + exit(-1); + } + + checkPort(serverIp, startPort, endPort, pktLen); + + return; +} + + + +static void taosNetTestServer(uint16_t startPort, uint16_t endPort, int pktLen) { + + int port = startPort; + int num = endPort - startPort + 1; + + if (num < 0) { + num = 1; + } + + pthread_t *pids = malloc(2 * num * sizeof(pthread_t)); + info_s * tinfos = malloc(num * sizeof(info_s)); + info_s * uinfos = malloc(num * sizeof(info_s)); + + for (size_t i = 0; i < num; i++) { + info_s *tcpInfo = tinfos + i; + tcpInfo->port = (uint16_t)(port + i); + tcpInfo->pktLen = pktLen; + + if (pthread_create(pids + i, NULL, bindTcpPort, tcpInfo) != 0) + { + printf("create thread fail, port:%d.\n", port); + exit(-1); + } + + info_s *udpInfo = uinfos + i; + udpInfo->port = (uint16_t)(port + i); + if (pthread_create(pids + num + i, NULL, bindUdpPort, udpInfo) != 0) + { + printf("create thread fail, port:%d.\n", port); + exit(-1); + } + } + + for (int i = 0; i < num; i++) { + pthread_join(pids[i], NULL); + pthread_join(pids[(num + i)], NULL); + } +} + + +void taosNetTest(const char* host, uint16_t port, uint16_t endPort, int pktLen, const char* netTestRole) { + if (pktLen > MAX_PKG_LEN) { + printf("test packet len overflow: %d, max len not greater than %d bytes\n", pktLen, MAX_PKG_LEN); + exit(-1); + } + + if (port && endPort) { + if (port > endPort) { + printf("endPort[%d] must not lesss port[%d]\n", endPort, port); + exit(-1); + } + } + + if (host && host[0] != 0) { + if (strlen(host) >= TSDB_EP_LEN) { + printf("host invalid: %s\n", host); + exit(-1); + } + + taosGetFqdnPortFromEp(host, serverFqdn, &g_startPort); + } else { + tstrncpy(serverFqdn, "127.0.0.1", TSDB_IPv4ADDR_LEN); + g_startPort = tsServerPort; + } + + if (port) { + g_startPort = port; + } + + if (endPort) { + g_endPort = endPort; + } + + if (port > endPort) { + printf("endPort[%d] must not lesss port[%d]\n", g_endPort, g_startPort); + exit(-1); + } + + if (0 == strcmp("client", netTestRole)) { + printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", serverFqdn, g_startPort, g_endPort, pktLen); + taosNetTestClient(serverFqdn, g_startPort, g_endPort, pktLen); + } else if (0 == strcmp("server", netTestRole)) { + taosNetTestServer(g_startPort, g_endPort, pktLen); + } +} + diff --git a/tests/examples/lua/lua_connector.c b/tests/examples/lua/lua_connector.c index f4065bb274..143f16a799 100644 --- a/tests/examples/lua/lua_connector.c +++ b/tests/examples/lua/lua_connector.c @@ -58,8 +58,10 @@ static int l_query(lua_State *L){ int table_index = lua_gettop(L); // printf("receive command:%s\r\n",s); - if(taos_query(taos, s)!=0){ - printf("failed, reason:%s\n", taos_errstr(taos)); + result = taos_query(taos,s); + int32_t code = taos_errno(result); + if( code != 0){ + printf("failed, reason:%s\n", taos_errstr(result)); lua_pushnumber(L, -1); lua_setfield(L, table_index, "code"); lua_pushstring(L, taos_errstr(taos)); @@ -69,24 +71,13 @@ static int l_query(lua_State *L){ }else{ //printf("success to query.\n"); - result = taos_use_result(taos); - - if (result == NULL) { - printf("failed to get result, reason:%s\n", taos_errstr(taos)); - lua_pushnumber(L, -2); - lua_setfield(L, table_index, "code"); - lua_pushstring(L, taos_errstr(taos)); - lua_setfield(L, table_index, "error"); - return 1; - } - TAOS_ROW row; int rows = 0; - int num_fields = taos_field_count(taos); + int num_fields = taos_field_count(result); TAOS_FIELD *fields = taos_fetch_fields(result); char temp[256]; - int affectRows = taos_affected_rows(taos); + int affectRows = taos_affected_rows(result); // printf(" affect rows:%d\r\n", affectRows); lua_pushnumber(L, 0); lua_setfield(L, table_index, "code"); @@ -155,15 +146,13 @@ static int l_query(lua_State *L){ } void stream_cb(void *param, TAOS_RES *result, TAOS_ROW row){ - struct cb_param* p = (struct cb_param*) param; TAOS_FIELD *fields = taos_fetch_fields(result); int numFields = taos_num_fields(result); + printf("\nnumfields:%d\n", numFields); printf("\n\r-----------------------------------------------------------------------------------\n"); - // printf("r:%d, L:%d\n",p->callback, p->state); - lua_State *L = p->state; lua_rawgeti(L, LUA_REGISTRYINDEX, p->callback); diff --git a/tests/examples/lua/test.lua b/tests/examples/lua/test.lua index 38ae1c82f2..4d5f9fe7d3 100644 --- a/tests/examples/lua/test.lua +++ b/tests/examples/lua/test.lua @@ -15,7 +15,7 @@ else conn = res.conn end -local res = driver.query(conn,"drop database demo") +local res = driver.query(conn,"drop database if exists demo") res = driver.query(conn,"create database demo") if res.code ~=0 then @@ -106,7 +106,7 @@ end --From now on we begin continous query in an definite (infinite if you want) loop. local loop_index = 0 -while loop_index < 20 do +while loop_index < 10 do local t = os.time()*1000 local v = loop_index res = driver.query(conn,string.format("INSERT INTO therm1 VALUES (%d, %d)",t,v)) diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index 8c8f44b016..0ecc227402 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -2350,7 +2350,7 @@ class ServiceManagerThread: self._thread2.start() # wait for service to start - for i in range(0, 10): + for i in range(0, 100): time.sleep(1.0) # self.procIpcBatch() # don't pump message during start up print("_zz_", end="", flush=True) @@ -2358,7 +2358,7 @@ class ServiceManagerThread: logger.info("[] TDengine service READY to process requests") return # now we've started # TODO: handle this better? - self.procIpcBatch(20, True) # display output before cronking out, trim to last 20 msgs, force output + self.procIpcBatch(100, True) # display output before cronking out, trim to last 20 msgs, force output raise RuntimeError("TDengine service did not start successfully") def stop(self): diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index fd5aa4ecf0..d600a003b0 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -12,7 +12,7 @@ python3 ./test.py -f insert/tinyint.py python3 ./test.py -f insert/date.py python3 ./test.py -f insert/binary.py python3 ./test.py -f insert/nchar.py -python3 ./test.py -f insert/nchar-boundary.py +#python3 ./test.py -f insert/nchar-boundary.py python3 ./test.py -f insert/nchar-unicode.py python3 ./test.py -f insert/multi.py python3 ./test.py -f insert/randomNullCommit.py @@ -20,7 +20,7 @@ python3 ./test.py -f insert/randomNullCommit.py python3 ./test.py -f table/column_name.py python3 ./test.py -f table/column_num.py python3 ./test.py -f table/db_table.py -python3 ./test.py -f table/tablename-boundary.py +#python3 ./test.py -f table/tablename-boundary.py # tag python3 ./test.py -f tag_lite/filter.py @@ -52,7 +52,7 @@ python3 ./test.py -f tag_lite/set.py python3 ./test.py -f tag_lite/smallint.py python3 ./test.py -f tag_lite/tinyint.py -python3 ./test.py -f dbmgmt/database-name-boundary.py +#python3 ./test.py -f dbmgmt/database-name-boundary.py python3 ./test.py -f import_merge/importBlock1HO.py python3 ./test.py -f import_merge/importBlock1HPO.py @@ -145,6 +145,8 @@ python3 ./test.py -f query/queryJoin.py python3 ./test.py -f query/select_last_crash.py python3 ./test.py -f query/queryNullValueTest.py python3 ./test.py -f query/queryInsertValue.py +python3 ./test.py -f query/queryConnection.py +python3 ./test.py -f query/natualInterval.py #stream python3 ./test.py -f stream/metric_1.py @@ -182,7 +184,7 @@ python3 ./test.py -f functions/function_spread.py python3 ./test.py -f functions/function_stddev.py python3 ./test.py -f functions/function_sum.py python3 ./test.py -f functions/function_top.py -python3 ./test.py -f functions/function_twa.py +#python3 ./test.py -f functions/function_twa.py # tools python3 test.py -f tools/taosdemo.py diff --git a/tests/pytest/query/queryConnection.py b/tests/pytest/query/queryConnection.py new file mode 100644 index 0000000000..ed05b5e6bd --- /dev/null +++ b/tests/pytest/query/queryConnection.py @@ -0,0 +1,52 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(50))") + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags("dev_01")') + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags("dev_02")') + + tdSql.execute( + """INSERT INTO dev_001(ts, tagtype) VALUES('2020-05-13 10:00:00.000', 1), + ('2020-05-13 10:00:00.001', 1) + dev_002 VALUES('2020-05-13 10:00:00.001', 1)""") + + for i in range(10): + for j in range(1000): + tdSql.query("select * from db.st") + tdLog.sleep(10) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/script/fullGeneralSuite.sim b/tests/script/fullGeneralSuite.sim index 15cc2954e8..d137e53d27 100644 --- a/tests/script/fullGeneralSuite.sim +++ b/tests/script/fullGeneralSuite.sim @@ -24,6 +24,7 @@ run general/compute/diff2.sim run general/compute/first.sim run general/compute/interval.sim run general/compute/last.sim +run general/compute/last_row.sim run general/compute/leastsquare.sim run general/compute/max.sim run general/compute/min.sim diff --git a/tests/script/general/compute/last_row.sim b/tests/script/general/compute/last_row.sim new file mode 100644 index 0000000000..cc5cc3edbb --- /dev/null +++ b/tests/script/general/compute/last_row.sim @@ -0,0 +1,175 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/exec.sh -n dnode1 -s start +sleep 3000 +sql connect + +$dbPrefix = m_la_db +$tbPrefix = m_la_tb +$mtPrefix = m_la_mt +$tbNum = 10 +$rowNum = 20 +$totalNum = 200 + +print =============== step1 +$i = 0 +$db = $dbPrefix . $i +$mt = $mtPrefix . $i + +sql drop database $db -x step1 +step1: +sql create database $db +sql use $db +sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int) + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mt tags( $i ) + + $x = 0 + while $x < $rowNum + $ms = $x . m + sql insert into $tb values (now + $ms , $x ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +sleep 100 + +print =============== step2 +$i = 1 +$tb = $tbPrefix . $i + +sql select last_row(tbcol) from $tb +print ===> $data00 +if $data00 != 19 then + return -1 +endi + +print =============== step3 +sql select last_row(tbcol) from $tb where ts < now + 4m +print ===> $data00 +if $data00 != 4 then + return -1 +endi + +print =============== step4 +sql select last_row(tbcol) as b from $tb +print ===> $data00 +if $data00 != 19 then + return -1 +endi + + + +print =============== step7 +sql select last_row(tbcol) from $mt +print ===> $data00 +if $data00 != 19 then + return -1 +endi + +print =============== step8 +sql select last_row(tbcol) as c from $mt where ts < now + 4m +print ===> $data00 +if $data00 != 4 then + return -1 +endi + +sql select last_row(tbcol) as c from $mt where tgcol < 5 +print ===> $data00 +if $data00 != 19 then + return -1 +endi + +sql select last_row(tbcol) as c from $mt where tgcol < 5 and ts < now + 4m +print ===> $data00 +if $data00 != 4 then + return -1 +endi + + + +print =============== step10 +sql select last_row(tbcol) as b from $mt group by tgcol +print ===> $data00 +if $data00 != 19 then + return -1 +endi + +if $rows != $tbNum then + return -1 +endi + +print =============== step11 + +sql insert into $tb values(now + 1h, 10) +sql insert into $tb values(now + 3h, null) +sql insert into $tb values(now + 5h, -1) +sql insert into $tb values(now + 7h, null) + +## for super table +sql select last_row(*) from $mt where ts < now + 6h +if $data01 != -1 then + return -1 +endi + +sql select last_row(*) from $mt where ts < now + 8h +if $data01 != NULL then + return -1 +endi + +sql select last_row(*) from $mt +if $data01 != NULL then + return -1 +endi + +sql select last_row(*) from $mt where ts < now + 4h +if $data01 != NULL then + return -1 +endi + +sql select last_row(*) from $mt where ts > now + 1h and ts < now + 4h +if $data01 != NULL then + return -1 +endi + +## for table +sql select last_row(*) from $tb where ts < now + 6h +if $data01 != -1 then + return -1 +endi + +sql select last_row(*) from $tb where ts < now + 8h +if $data01 != NULL then + return -1 +endi + +sql select last_row(*) from $tb +if $data01 != NULL then + return -1 +endi + +sql select last_row(*) from $tb where ts < now + 4h +if $data01 != NULL then + return -1 +endi + +sql select last_row(*) from $tb where ts > now + 1h and ts < now + 4h +if $data01 != NULL then + return -1 +endi + +print =============== clear +sql drop database $db +sql show databases +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index 88e7dece4c..b7f98e49e0 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -119,7 +119,7 @@ endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' used1' 127.0.0.1:7111/rest/sql print 17-> $system_content -if $system_content != @{"status":"error","code":512,"desc":"invalid SQL: invalid SQL: syntax error near 'used1'"}@ then +if $system_content != @{"status":"error","code":534,"desc":"Syntax errr in SQL"}@ then return -1 endi @@ -230,4 +230,4 @@ if $system_content != @{"status":"succ","head":["ts","speed"],"data":[["2017-12- return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index 1bb0ff5448..f17e28c1da 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -482,15 +482,31 @@ sql insert into um2 using m2 tags(9) values(1000001, 10)(2000000, 20); sql_error select count(*) from m1,m2 where m1.a=m2.a and m1.ts=m2.ts; -#empty table join test, add for no result join test +print ====> empty table/empty super-table join test, add for no result join test sql create database ux1; sql use ux1; sql create table m1(ts timestamp, k int) tags(a binary(12), b int); sql create table tm0 using m1 tags('abc', 1); sql create table m2(ts timestamp, k int) tags(a int, b binary(12)); + +sql select count(*) from m1, m2 where m1.ts=m2.ts and m1.b=m2.a; +if $rows != 0 then + return -1 +endi + sql create table tm2 using m2 tags(2, 'abc'); sql select count(*) from tm0, tm2 where tm0.ts=tm2.ts; -sql select count(*) from m1, m2 where m1.ts=m2.ts and m1.b=m2.a +if $rows != 0 then + return -1 +endi + +sql select count(*) from m1, m2 where m1.ts=m2.ts and m1.b=m2.a; +if $rows != 0 then + return -1 +endi + +sql drop table tm2; +sql select count(*) from m1, m2 where m1.ts=m2.ts and m1.b=m2.a; sql drop database ux1; system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tmp/182.sim b/tests/script/tmp/182.sim index 27e064dc9b..a178282cf8 100644 --- a/tests/script/tmp/182.sim +++ b/tests/script/tmp/182.sim @@ -32,9 +32,10 @@ system sh/cfg.sh -n dnode2 -c http -v 1 system sh/cfg.sh -n dnode3 -c http -v 1 system sh/cfg.sh -n dnode4 -c http -v 1 +system sh/cfg.sh -n dnode1 -c httpMaxThreads -v 4 system sh/cfg.sh -n dnode1 -c firstEp -v 127.0.0.1:6030 system sh/cfg.sh -n dnode1 -c secondEp -v 127.0.0.1:6030 system sh/cfg.sh -n dnode1 -c serverPort -v 6030 system sh/cfg.sh -n dnode1 -c fqdn -v 127.0.0.1 -#system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode1 -s start diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 536727100d..e1fedaee3c 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -28,6 +28,6 @@ IF (TD_LINUX) #add_executable(createNormalTable createNormalTable.c) #target_link_libraries(createNormalTable taos_static tutil common pthread) - #add_executable(queryPerformance queryPerformance.c) - #target_link_libraries(queryPerformance taos_static tutil common pthread) + add_executable(queryPerformance queryPerformance.c) + target_link_libraries(queryPerformance taos_static tutil common pthread) ENDIF() diff --git a/tests/test/c/queryPerformance.c b/tests/test/c/queryPerformance.c index 5e7a4333de..eda082dd4f 100644 --- a/tests/test/c/queryPerformance.c +++ b/tests/test/c/queryPerformance.c @@ -34,27 +34,41 @@ typedef struct { void *syncTest(void *param); void shellParseArgument(int argc, char *argv[]); -void insertData(); +void queryData(); -int64_t numOfThreads = 100; -char sql[10240] = "show dnodes"; -int32_t loopTimes = 1000; +int numOfThreads = 10; +int useGlobalConn = 1; +int requestPerThread = 10000; +char requestSql[10240] = "show dnodes"; +TAOS *globalConn; int main(int argc, char *argv[]) { shellParseArgument(argc, argv); taos_init(); - insertData(); + queryData(); } -void insertData() { +void queryData() { struct timeval systemTime; int64_t st, et; + char fqdn[TSDB_FQDN_LEN]; + uint16_t port; + + if (useGlobalConn) { + taosGetFqdnPortFromEp(tsFirst, fqdn, &port); + + globalConn = taos_connect(fqdn, "root", "taosdata", NULL, port); + if (globalConn == NULL) { + pError("failed to connect to DB, reason:%s", taos_errstr(globalConn)); + exit(1); + } + } + + pPrint("%d threads are spawned to query", numOfThreads); gettimeofday(&systemTime, NULL); st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - pPrint("%" PRId64 " threads are spawned to query", numOfThreads); - pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); @@ -73,41 +87,41 @@ void insertData() { gettimeofday(&systemTime, NULL); et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - double mseconds = (et - st) / 1000.0; + double totalTimeMs = (et - st) / 1000.0; - int64_t request = loopTimes * numOfThreads; - float avg = mseconds / request;; - float qps = 1000 / avg * numOfThreads; - - pPrint( - "%sall threads:%ld finished, use %.1lf ms, qps:%f, avg:%f %s", - GREEN, numOfThreads, mseconds, qps, avg, NC); + int totalReq = requestPerThread * numOfThreads; + float rspTime = totalTimeMs / requestPerThread; + float qps = totalReq / (totalTimeMs / 1000); - pPrint("threads exit"); + pPrint("%s threads:%d, totalTime %.1fms totalReq:%d qps:%.1f rspTime:%.3fms %s", GREEN, numOfThreads, totalTimeMs, + totalReq, qps, rspTime, NC); pthread_attr_destroy(&thattr); free(pInfo); } void *syncTest(void *param) { - TAOS * con; - SInfo * pInfo = (SInfo *)param; - struct timeval systemTime; - - pPrint("thread:%d, start to run", pInfo->threadIndex); + TAOS * con; + SInfo * pInfo = (SInfo *)param; char fqdn[TSDB_FQDN_LEN]; uint16_t port; - taosGetFqdnPortFromEp(tsFirst, fqdn, &port); + if (useGlobalConn) { + pPrint("thread:%d, start to run use global connection", pInfo->threadIndex); + con = globalConn; + } else { + pPrint("thread:%d, start to run, and create new conn", pInfo->threadIndex); + taosGetFqdnPortFromEp(tsFirst, fqdn, &port); - con = taos_connect(fqdn, "root", "taosdata", NULL, port); - if (con == NULL) { - pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); - exit(1); + con = taos_connect(fqdn, "root", "taosdata", NULL, port); + if (con == NULL) { + pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); + exit(1); + } } - for (int i = 0; i < loopTimes; ++i) { - void *tres = taos_query(con, sql); + for (int i = 0; i < requestPerThread; ++i) { + void *tres = taos_query(con, requestSql); TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { @@ -117,13 +131,10 @@ void *syncTest(void *param) { do { row = taos_fetch_row(tres); - } while( row != NULL); + } while (row != NULL); taos_free_result(tres); } - - gettimeofday(&systemTime, NULL); - return NULL; } @@ -134,12 +145,14 @@ void printHelp() { printf("%s%s\n", indent, "-c"); printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); printf("%s%s\n", indent, "-s"); - printf("%s%s%s%s\n", indent, indent, "The sql to be executed, default is %s", sql); - printf("%s%s\n", indent, "-l"); - printf("%s%s%s%d\n", indent, indent, "Loop Times per thread, default is ", loopTimes); + printf("%s%s%s%s\n", indent, indent, "The sql to be executed, default is ", requestSql); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%d\n", indent, indent, "Request per thread, default is ", requestPerThread); printf("%s%s\n", indent, "-t"); - printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of threads to be used, default is ", numOfThreads); - + printf("%s%s%s%d\n", indent, indent, "Number of threads to be used, default is ", numOfThreads); + printf("%s%s\n", indent, "-g"); + printf("%s%s%s%d\n", indent, indent, "Whether to share connections between threads, default is ", useGlobalConn); + exit(EXIT_SUCCESS); } @@ -151,17 +164,20 @@ void shellParseArgument(int argc, char *argv[]) { } else if (strcmp(argv[i], "-c") == 0) { strcpy(configDir, argv[++i]); } else if (strcmp(argv[i], "-s") == 0) { - strcpy(sql, argv[++i]); - } else if (strcmp(argv[i], "-l") == 0) { - loopTimes = atoi(argv[++i]); + strcpy(requestSql, argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + requestPerThread = atoi(argv[++i]); } else if (strcmp(argv[i], "-t") == 0) { numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-g") == 0) { + useGlobalConn = atoi(argv[++i]); } else { } } - pPrint("%ssql:%s%s", GREEN, sql, NC); - pPrint("%sloopTImes:%d%s", GREEN, loopTimes, NC); - pPrint("%snumOfThreads:%" PRId64 "%s", GREEN, numOfThreads, NC); - pPrint("%sstart to run%s", GREEN, NC); + pPrint("%s sql:%s %s", GREEN, requestSql, NC); + pPrint("%s requestPerThread:%d %s", GREEN, requestPerThread, NC); + pPrint("%s numOfThreads:%d %s", GREEN, numOfThreads, NC); + pPrint("%s useGlobalConn:%d %s", GREEN, useGlobalConn, NC); + pPrint("%s start to run %s", GREEN, NC); }