diff --git a/documentation20/cn/02.getting-started/docs.md b/documentation20/cn/02.getting-started/docs.md index fa36481646..c67b9e61b9 100644 --- a/documentation20/cn/02.getting-started/docs.md +++ b/documentation20/cn/02.getting-started/docs.md @@ -126,7 +126,7 @@ taos> source ; $ taosdemo ``` -该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "t0" 到 "t9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupdId，groupdId 被设置为 1 到 10， location 被设置为 "beijing" 或者 "shanghai"。 +该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupdId，groupdId 被设置为 1 到 10， location 被设置为 "beijing" 或者 "shanghai"。 执行这条命令大概需要几分钟，最后共插入 1 亿条记录。 @@ -156,10 +156,10 @@ taos> select count(*) from test.meters where location="beijing"; taos> select avg(current), max(voltage), min(phase) from test.meters where groupdId=10; ``` -- 对表 t10 按 10s 进行平均值、最大值和最小值聚合统计： +- 对表 d10 按 10s 进行平均值、最大值和最小值聚合统计： ```mysql -taos> select avg(current), max(voltage), min(phase) from test.t10 interval(10s); +taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s); ``` **Note:** taosdemo 命令本身带有很多选项，配置表的数目、记录条数等等，请执行 `taosdemo --help` 详细列出。您可以设置不同参数进行体验。 diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 9044f23672..aa09013e53 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -182,7 +182,13 @@ function install_jemalloc() { ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi - ${csudo} ldconfig + + if [ -d /etc/ld.so.conf.d ]; then + ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf + ${csudo} ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi fi } diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 2c36e85970..a7dc931882 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -41,10 +41,10 @@ fi if [ "$osType" != "Darwin" ]; then if [ "$pagMode" == "lite" ]; then - #strip ${build_dir}/bin/taosd + #strip ${build_dir}/bin/taosd strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh" - else + else bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo \ ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh ${script_dir}/taosd-dump-cfg.gdb" fi @@ -139,7 +139,7 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp -r ${examples_dir}/C# ${install_dir}/examples fi # Copy driver -mkdir -p ${install_dir}/driver +mkdir -p ${install_dir}/driver cp ${lib_files} ${install_dir}/driver # Copy connector @@ -168,7 +168,7 @@ fi # exit 1 -cd ${release_dir} +cd ${release_dir} # install_dir has been distinguishes cluster from edege, so comments this code pkg_name=${install_dir}-${osType}-${cpuType} @@ -195,6 +195,15 @@ if [ "$pagMode" == "lite" ]; then pkg_name=${pkg_name}-Lite fi +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stable or beta" + exit 1 +fi + if [ "$osType" != "Darwin" ]; then tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : else diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh index 6d10245b4f..89591cac23 100755 --- a/packaging/tools/makeclient_power.sh +++ b/packaging/tools/makeclient_power.sh @@ -41,10 +41,10 @@ fi if [ "$osType" != "Darwin" ]; then # if [ "$pagMode" == "lite" ]; then -# strip ${build_dir}/bin/powerd +# strip ${build_dir}/bin/powerd # strip ${build_dir}/bin/power # bin_files="${build_dir}/bin/power ${script_dir}/remove_client_power.sh" -# else +# else # bin_files="${build_dir}/bin/power ${build_dir}/bin/powerdemo ${script_dir}/remove_client_power.sh ${script_dir}/set_core.sh" # fi lib_files="${build_dir}/lib/libtaos.so.${version}" @@ -67,6 +67,39 @@ mkdir -p ${install_dir} mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg +if [ -f ${build_dir}/bin/jemalloc-config ]; then + mkdir -p ${install_dir}/jemalloc/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3} + cp ${build_dir}/bin/jemalloc-config ${install_dir}/jemalloc/bin + if [ -f ${build_dir}/bin/jemalloc.sh ]; then + cp ${build_dir}/bin/jemalloc.sh ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/bin/jeprof ]; then + cp ${build_dir}/bin/jeprof ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/include/jemalloc/jemalloc.h ]; then + cp ${build_dir}/include/jemalloc/jemalloc.h ${install_dir}/jemalloc/include/jemalloc + fi + if [ -f ${build_dir}/lib/libjemalloc.so.2 ]; then + cp ${build_dir}/lib/libjemalloc.so.2 ${install_dir}/jemalloc/lib + ln -sf libjemalloc.so.2 ${install_dir}/jemalloc/lib/libjemalloc.so + fi + if [ -f ${build_dir}/lib/libjemalloc.a ]; then + cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then + cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/pkgconfig/jemalloc.pc ]; then + cp ${build_dir}/lib/pkgconfig/jemalloc.pc ${install_dir}/jemalloc/lib/pkgconfig + fi + if [ -f ${build_dir}/share/doc/jemalloc/jemalloc.html ]; then + cp ${build_dir}/share/doc/jemalloc/jemalloc.html ${install_dir}/jemalloc/share/doc/jemalloc + fi + if [ -f ${build_dir}/share/man/man3/jemalloc.3 ]; then + cp ${build_dir}/share/man/man3/jemalloc.3 ${install_dir}/jemalloc/share/man/man3 + fi +fi + sed -i '/dataDir/ {s/taos/power/g}' ${install_dir}/cfg/taos.cfg sed -i '/logDir/ {s/taos/power/g}' ${install_dir}/cfg/taos.cfg sed -i "s/TDengine/PowerDB/g" ${install_dir}/cfg/taos.cfg @@ -77,11 +110,11 @@ if [ "$osType" != "Darwin" ]; then strip ${build_dir}/bin/taos cp ${build_dir}/bin/taos ${install_dir}/bin/power cp ${script_dir}/remove_power.sh ${install_dir}/bin - else + else cp ${build_dir}/bin/taos ${install_dir}/bin/power cp ${script_dir}/remove_power.sh ${install_dir}/bin - cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo - cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump + cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo + cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump cp ${script_dir}/set_core.sh ${install_dir}/bin cp ${script_dir}/get_client.sh ${install_dir}/bin cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin @@ -158,15 +191,15 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp -r ${examples_dir}/JDBC ${install_dir}/examples cp -r ${examples_dir}/matlab ${install_dir}/examples sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/examples/matlab/TDengineDemo.m - cp -r ${examples_dir}/python ${install_dir}/examples + cp -r ${examples_dir}/python ${install_dir}/examples sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/examples/python/read_example.py cp -r ${examples_dir}/R ${install_dir}/examples sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/examples/R/command.txt - cp -r ${examples_dir}/go ${install_dir}/examples + cp -r ${examples_dir}/go ${install_dir}/examples sed -i '/root/ {s/taosdata/powerdb/g}' ${install_dir}/examples/go/taosdemo.go fi # Copy driver -mkdir -p ${install_dir}/driver +mkdir -p ${install_dir}/driver cp ${lib_files} ${install_dir}/driver # Copy connector @@ -188,11 +221,11 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then echo "WARNING: go connector not found, please check if want to use it!" fi cp -r ${connector_dir}/python ${install_dir}/connector - + sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/cinterface.py - + sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/subscription.py - + sed -i '/self._password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/connection.py fi # Copy release note @@ -200,7 +233,7 @@ fi # exit 1 -cd ${release_dir} +cd ${release_dir} if [ "$verMode" == "cluster" ]; then pkg_name=${install_dir}-${osType}-${cpuType} @@ -217,8 +250,8 @@ fi if [ "$verType" == "beta" ]; then pkg_name=${pkg_name}-${verType} -elif [ "$verType" == "stable" ]; then - pkg_name=${pkg_name} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} else echo "unknow verType, nor stable or beta" exit 1 diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 806d5a62df..abc3d47e0b 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -94,11 +94,21 @@ typedef struct SVgroupTableInfo { SArray *itemList; // SArray } SVgroupTableInfo; +typedef struct SBlockKeyTuple { + TSKEY skey; + void* payloadAddr; +} SBlockKeyTuple; + +typedef struct SBlockKeyInfo { + int32_t maxBytesAlloc; + SBlockKeyTuple* pKeyTuple; +} SBlockKeyInfo; + int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len); int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta); -void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf); +int tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo); void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo); void doRetrieveSubqueryData(SSchedMsg *pMsg); @@ -309,10 +319,9 @@ bool hasMoreClauseToTry(SSqlObj* pSql); void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta); void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); -void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows); void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp); int tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corEpSet); -int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp); +int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp, bool metaClone); int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t length, SArray* pNameArray); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index bab335154f..7da7e17374 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -88,13 +88,43 @@ typedef struct SBoundColumn { int32_t offset; // all column offset value } SBoundColumn; +typedef struct { + uint16_t schemaColIdx; + uint16_t boundIdx; + uint16_t finalIdx; +} SBoundIdxInfo; + +typedef enum _COL_ORDER_STATUS { + ORDER_STATUS_UNKNOWN = 0, + ORDER_STATUS_ORDERED = 1, + ORDER_STATUS_DISORDERED = 2, +} EOrderStatus; + typedef struct SParsedDataColInfo { - int16_t numOfCols; - int16_t numOfBound; - int32_t *boundedColumns; - SBoundColumn *cols; + int16_t numOfCols; + int16_t numOfBound; + int32_t * boundedColumns; // bounded column idx according to schema + SBoundColumn * cols; + SBoundIdxInfo *colIdxInfo; + int8_t orderStatus; // bounded columns: } SParsedDataColInfo; +#define IS_DATA_COL_ORDERED(s) ((s) == (int8_t)ORDER_STATUS_ORDERED) + +typedef struct { + SSchema * pSchema; + int16_t sversion; + int32_t flen; + uint16_t nCols; + void * buf; + void * pDataBlock; + SSubmitBlk *pSubmitBlk; +} SMemRowBuilder; + +typedef struct { + TDRowLenT allNullLen; +} SMemRowHelper; + typedef struct STableDataBlocks { SName tableName; int8_t tsSource; // where does the UNIX timestamp come from, server or client @@ -109,12 +139,13 @@ typedef struct STableDataBlocks { STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache char *pData; - SParsedDataColInfo boundColumnInfo; + SParsedDataColInfo boundColumnInfo; // for parameter ('?') binding - uint32_t numOfAllocedParams; - uint32_t numOfParams; - SParamInfo *params; + uint32_t numOfAllocedParams; + uint32_t numOfParams; + SParamInfo * params; + SMemRowHelper rowHelper; } STableDataBlocks; typedef struct { @@ -243,6 +274,7 @@ typedef struct SSqlObj { void * pStream; void * pSubscription; char * sqlstr; + void * pBuf; // table meta buffer char parseRetry; char retry; char maxRetry; @@ -383,12 +415,17 @@ extern int tscRefId; extern int tscNumOfObj; // number of existed sqlObj in current process. extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); - + void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables); int16_t getNewResColId(SSqlCmd* pCmd); +int32_t schemaIdxCompar(const void *lhs, const void *rhs); +int32_t boundIdxCompar(const void *lhs, const void *rhs); +int initSMemRowHelper(SMemRowHelper *pHelper, SSchema *pSSchema, uint16_t nCols, uint16_t allNullColsLen); +int32_t getExtendedRowSize(STableComInfo *tinfo); + #ifdef __cplusplus } #endif -#endif +#endif \ No newline at end of file diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 15a157816f..87ef82bd0e 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -38,10 +38,33 @@ enum { TSDB_USE_CLI_TS = 1, }; +static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; +static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; + static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t *numOfRows); static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDataColInfo *pColInfo, SSchema *pSchema, char *str, char **end); +int32_t getExtendedRowSize(STableComInfo *tinfo) { + return tinfo->rowSize + PAYLOAD_HEADER_LEN + PAYLOAD_COL_HEAD_LEN * tinfo->numOfColumns; +} +int initSMemRowHelper(SMemRowHelper *pHelper, SSchema *pSSchema, uint16_t nCols, uint16_t allNullColsLen) { + pHelper->allNullLen = allNullColsLen; // TODO: get allNullColsLen when creating or altering table meta + if (pHelper->allNullLen == 0) { + for (uint16_t i = 0; i < nCols; ++i) { + uint8_t type = pSSchema[i].type; + int32_t typeLen = TYPE_BYTES[type]; + pHelper->allNullLen += typeLen; + if (TSDB_DATA_TYPE_BINARY == type) { + pHelper->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); + } else if (TSDB_DATA_TYPE_NCHAR == type) { + int len = VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE; + pHelper->allNullLen += len; + } + } + } + return 0; +} static int32_t tscToDouble(SStrToken *pToken, double *value, char **endPtr) { errno = 0; *value = strtold(pToken->z, endPtr); @@ -378,6 +401,342 @@ int32_t tsParseOneColumn(SSchema *pSchema, SStrToken *pToken, char *payload, cha return TSDB_CODE_SUCCESS; } +static FORCE_INLINE TDRowLenT tsSetPayloadColValue(char *payloadStart, char *payload, int16_t columnId, + uint8_t columnType, const void *value, uint16_t valueLen, TDRowTLenT tOffset) { + payloadColSetId(payload, columnId); + payloadColSetType(payload, columnType); + memcpy(POINTER_SHIFT(payloadStart,tOffset), value, valueLen); + return valueLen; +} + +static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *payloadStart, char *primaryKeyStart, + char *payload, char *msg, char **str, bool primaryKey, int16_t timePrec, + TDRowTLenT tOffset, TDRowLenT *sizeAppend, TDRowLenT *dataRowColDeltaLen, + TDRowLenT *kvRowColLen) { + int64_t iv; + int32_t ret; + char * endptr = NULL; + + if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { + return tscInvalidOperationMsg(msg, "invalid numeric data", pToken->z); + } + + switch (pSchema->type) { + case TSDB_DATA_TYPE_BOOL: { // bool + if (isNullStr(pToken)) { + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_BOOL), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); + } else { + if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { + if (strncmp(pToken->z, "true", pToken->n) == 0) { + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &TRUE_VALUE, + TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); + } else if (strncmp(pToken->z, "false", pToken->n) == 0) { + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &FALSE_VALUE, + TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); + } else { + return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z); + } + } else if (pToken->type == TK_INTEGER) { + iv = strtoll(pToken->z, NULL, 10); + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); + } else if (pToken->type == TK_FLOAT) { + double dv = strtod(pToken->z, NULL); + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); + } else { + return tscInvalidOperationMsg(msg, "invalid bool data", pToken->z); + } + } + break; + } + + case TSDB_DATA_TYPE_TINYINT: + if (isNullStr(pToken)) { + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_TINYINT), TYPE_BYTES[TSDB_DATA_TYPE_TINYINT], tOffset); + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid tinyint data", pToken->z); + } else if (!IS_VALID_TINYINT(iv)) { + return tscInvalidOperationMsg(msg, "data overflow", pToken->z); + } + + uint8_t tmpVal = (uint8_t)iv; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_TINYINT], tOffset); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TINYINT]); + } + + break; + + case TSDB_DATA_TYPE_UTINYINT: + if (isNullStr(pToken)) { + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_UTINYINT), TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT], tOffset); + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid unsigned tinyint data", pToken->z); + } else if (!IS_VALID_UTINYINT(iv)) { + return tscInvalidOperationMsg(msg, "unsigned tinyint data overflow", pToken->z); + } + + uint8_t tmpVal = (uint8_t)iv; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT], tOffset); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT]); + } + + break; + + case TSDB_DATA_TYPE_SMALLINT: + if (isNullStr(pToken)) { + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_SMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT], tOffset); + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid smallint data", pToken->z); + } else if (!IS_VALID_SMALLINT(iv)) { + return tscInvalidOperationMsg(msg, "smallint data overflow", pToken->z); + } + + int16_t tmpVal = (int16_t)iv; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT], tOffset); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT]); + } + + break; + + case TSDB_DATA_TYPE_USMALLINT: + if (isNullStr(pToken)) { + *sizeAppend = + tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_USMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT], tOffset); + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid unsigned smallint data", pToken->z); + } else if (!IS_VALID_USMALLINT(iv)) { + return tscInvalidOperationMsg(msg, "unsigned smallint data overflow", pToken->z); + } + + uint16_t tmpVal = (uint16_t)iv; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT], tOffset); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT]); + } + + break; + + case TSDB_DATA_TYPE_INT: + if (isNullStr(pToken)) { + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_INT), TYPE_BYTES[TSDB_DATA_TYPE_INT], tOffset); + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid int data", pToken->z); + } else if (!IS_VALID_INT(iv)) { + return tscInvalidOperationMsg(msg, "int data overflow", pToken->z); + } + + int32_t tmpVal = (int32_t)iv; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_INT], tOffset); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_INT]); + } + + break; + + case TSDB_DATA_TYPE_UINT: + if (isNullStr(pToken)) { + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_UINT), TYPE_BYTES[TSDB_DATA_TYPE_UINT], tOffset); + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid unsigned int data", pToken->z); + } else if (!IS_VALID_UINT(iv)) { + return tscInvalidOperationMsg(msg, "unsigned int data overflow", pToken->z); + } + + uint32_t tmpVal = (uint32_t)iv; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_UINT], tOffset); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UINT]); + } + + break; + + case TSDB_DATA_TYPE_BIGINT: + if (isNullStr(pToken)) { + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_BIGINT), TYPE_BYTES[TSDB_DATA_TYPE_BIGINT], tOffset); + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid bigint data", pToken->z); + } else if (!IS_VALID_BIGINT(iv)) { + return tscInvalidOperationMsg(msg, "bigint data overflow", pToken->z); + } + + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &iv, + TYPE_BYTES[TSDB_DATA_TYPE_BIGINT], tOffset); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BIGINT]); + } + break; + + case TSDB_DATA_TYPE_UBIGINT: + if (isNullStr(pToken)) { + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_UBIGINT), TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT], tOffset); + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid unsigned bigint data", pToken->z); + } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { + return tscInvalidOperationMsg(msg, "unsigned bigint data overflow", pToken->z); + } + + uint64_t tmpVal = (uint64_t)iv; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT], tOffset); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT]); + } + break; + + case TSDB_DATA_TYPE_FLOAT: + if (isNullStr(pToken)) { + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + getNullValue(TSDB_DATA_TYPE_FLOAT), TYPE_BYTES[TSDB_DATA_TYPE_FLOAT], tOffset); + } else { + double dv; + if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { + return tscInvalidOperationMsg(msg, "illegal float data", pToken->z); + } + + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || + isnan(dv)) { + return tscInvalidOperationMsg(msg, "illegal float data", pToken->z); + } + + float tmpVal = (float)dv; + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_FLOAT], tOffset); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_FLOAT]); + } + break; + + case TSDB_DATA_TYPE_DOUBLE: + if (isNullStr(pToken)) { + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_DOUBLE), TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE], tOffset); + } else { + double dv; + if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { + return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); + } + + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { + return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); + } + + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &dv, + TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE], tOffset); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE]); + } + break; + + case TSDB_DATA_TYPE_BINARY: + // binary data cannot be null-terminated char string, otherwise the last char of the string is lost + if (pToken->type == TK_NULL) { + payloadColSetId(payload, pSchema->colId); + payloadColSetType(payload, pSchema->type); + memcpy(POINTER_SHIFT(payloadStart, tOffset), tdGetNullVal(TSDB_DATA_TYPE_BINARY), VARSTR_HEADER_SIZE + CHAR_BYTES); + *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + CHAR_BYTES); + } else { // too long values will return invalid sql, not be truncated automatically + if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor + return tscInvalidOperationMsg(msg, "string data overflow", pToken->z); + } + // STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n); + + payloadColSetId(payload, pSchema->colId); + payloadColSetType(payload, pSchema->type); + varDataSetLen(POINTER_SHIFT(payloadStart,tOffset), pToken->n); + memcpy(varDataVal(POINTER_SHIFT(payloadStart,tOffset)), pToken->z, pToken->n); + *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + pToken->n); + *dataRowColDeltaLen += (TDRowLenT)(pToken->n - CHAR_BYTES); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + VARSTR_HEADER_SIZE + pToken->n); + } + + break; + + case TSDB_DATA_TYPE_NCHAR: + if (pToken->type == TK_NULL) { + payloadColSetId(payload, pSchema->colId); + payloadColSetType(payload, pSchema->type); + memcpy(POINTER_SHIFT(payloadStart,tOffset), tdGetNullVal(TSDB_DATA_TYPE_NCHAR), VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); + *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); + } else { + // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' + int32_t output = 0; + payloadColSetId(payload, pSchema->colId); + payloadColSetType(payload, pSchema->type); + if (!taosMbsToUcs4(pToken->z, pToken->n, varDataVal(POINTER_SHIFT(payloadStart,tOffset)), + pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { + char buf[512] = {0}; + snprintf(buf, tListLen(buf), "%s", strerror(errno)); + return tscInvalidOperationMsg(msg, buf, pToken->z); + } + + varDataSetLen(POINTER_SHIFT(payloadStart,tOffset), output); + + *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + output); + *dataRowColDeltaLen += (TDRowLenT)(output - sizeof(uint32_t)); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + VARSTR_HEADER_SIZE + output); + } + break; + + case TSDB_DATA_TYPE_TIMESTAMP: { + if (pToken->type == TK_NULL) { + if (primaryKey) { + // When building SKVRow primaryKey, we should not skip even with NULL value. + int64_t tmpVal = 0; + *sizeAppend = tsSetPayloadColValue(payloadStart, primaryKeyStart, pSchema->colId, pSchema->type, &tmpVal, + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], tOffset); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); + } else { + *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, + tdGetNullVal(TSDB_DATA_TYPE_TIMESTAMP), + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], tOffset); + } + } else { + int64_t tmpVal; + if (tsParseTime(pToken, &tmpVal, str, msg, timePrec) != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid timestamp", pToken->z); + } + + *sizeAppend = tsSetPayloadColValue(payloadStart, primaryKey ? primaryKeyStart : payload, pSchema->colId, + pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], tOffset); + *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); + } + + break; + } + } + + return TSDB_CODE_SUCCESS; +} + /* * The server time/client time should not be mixed up in one sql string * Do not employ sort operation is not involved if server time is used. @@ -414,24 +773,38 @@ int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start) { return TSDB_CODE_SUCCESS; } -int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, int32_t *len, - char *tmpTokenBuf, SInsertStatementParam* pInsertParam) { - int32_t index = 0; - SStrToken sToken = {0}; - char *payload = pDataBlocks->pData + pDataBlocks->size; +int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, int32_t *len, char *tmpTokenBuf, + SInsertStatementParam *pInsertParam) { + int32_t index = 0; + SStrToken sToken = {0}; + + SMemRowHelper *pHelper = &pDataBlocks->rowHelper; + char * payload = pDataBlocks->pData + pDataBlocks->size; SParsedDataColInfo *spd = &pDataBlocks->boundColumnInfo; - SSchema *schema = tscGetTableSchema(pDataBlocks->pTableMeta); + SSchema * schema = tscGetTableSchema(pDataBlocks->pTableMeta); + + TDRowTLenT dataRowLen = pHelper->allNullLen; + TDRowTLenT kvRowLen = TD_MEM_ROW_KV_VER_SIZE; + TDRowTLenT payloadValOffset = 0; + TDRowLenT colValOffset = 0; + ASSERT(dataRowLen > 0); + + payloadSetNCols(payload, spd->numOfBound); + payloadValOffset = payloadValuesOffset(payload); // rely on payloadNCols + // payloadSetTLen(payload, payloadValOffset); + + char *kvPrimaryKeyStart = payload + PAYLOAD_HEADER_LEN; // primaryKey in 1st column tuple + char *kvStart = kvPrimaryKeyStart + PAYLOAD_COL_HEAD_LEN; // the column tuple behind the primaryKey // 1. set the parsed value from sql string - int32_t rowSize = 0; for (int i = 0; i < spd->numOfBound; ++i) { // the start position in data block buffer of current value in sql int32_t colIndex = spd->boundedColumns[i]; - char *start = payload + spd->cols[colIndex].offset; - SSchema *pSchema = &schema[colIndex]; - rowSize += pSchema->bytes; + char *start = payload + spd->cols[colIndex].offset; + + SSchema *pSchema = &schema[colIndex]; // get colId here index = 0; sToken = tStrGetToken(*str, &index, true); @@ -453,7 +826,8 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i int16_t type = sToken.type; if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL && - type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || (sToken.n == 0) || (type == TK_RP)) { + type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || + (sToken.n == 0) || (type == TK_RP)) { return tscSQLSyntaxErrMsg(pInsertParam->msg, "invalid data or symbol", sToken.z); } @@ -467,10 +841,10 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i if (sToken.n >= TSDB_MAX_BYTES_PER_ROW) { return tscSQLSyntaxErrMsg(pInsertParam->msg, "too long string", sToken.z); } - + for (uint32_t k = 1; k < sToken.n - 1; ++k) { if (sToken.z[k] == '\\' || (sToken.z[k] == delim && sToken.z[k + 1] == delim)) { - tmpTokenBuf[j] = sToken.z[k + 1]; + tmpTokenBuf[j] = sToken.z[k + 1]; cnt++; j++; @@ -487,42 +861,54 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i sToken.n -= 2 + cnt; } - bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); - int32_t ret = tsParseOneColumn(pSchema, &sToken, start, pInsertParam->msg, str, isPrimaryKey, timePrec); + bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); + TDRowLenT dataRowDeltaColLen = 0; // When combine the data as SDataRow, the delta len between all NULL columns. + TDRowLenT kvRowColLen = 0; + TDRowLenT colValAppended = 0; + + if (!IS_DATA_COL_ORDERED(spd->orderStatus)) { + ASSERT(spd->colIdxInfo != NULL); + if(!isPrimaryKey) { + kvStart = POINTER_SHIFT(kvPrimaryKeyStart, spd->colIdxInfo[i].finalIdx * PAYLOAD_COL_HEAD_LEN); + } else { + ASSERT(spd->colIdxInfo[i].finalIdx == 0); + } + } + // the primary key locates in 1st column + int32_t ret = tsParseOneColumnKV(pSchema, &sToken, payload, kvPrimaryKeyStart, kvStart, pInsertParam->msg, str, + isPrimaryKey, timePrec, payloadValOffset + colValOffset, &colValAppended, + &dataRowDeltaColLen, &kvRowColLen); if (ret != TSDB_CODE_SUCCESS) { return ret; } - if (isPrimaryKey && tsCheckTimestamp(pDataBlocks, start) != TSDB_CODE_SUCCESS) { - tscInvalidOperationMsg(pInsertParam->msg, "client time/server time can not be mixed up", sToken.z); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - } - - // 2. set the null value for the columns that do not assign values - if (spd->numOfBound < spd->numOfCols) { - char *ptr = payload; - - for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (!spd->cols[i].hasVal) { // current column do not have any value to insert, set it to null - if (schema[i].type == TSDB_DATA_TYPE_BINARY) { - varDataSetLen(ptr, sizeof(int8_t)); - *(uint8_t*) varDataVal(ptr) = TSDB_DATA_BINARY_NULL; - } else if (schema[i].type == TSDB_DATA_TYPE_NCHAR) { - varDataSetLen(ptr, sizeof(int32_t)); - *(uint32_t*) varDataVal(ptr) = TSDB_DATA_NCHAR_NULL; - } else { - setNull(ptr, schema[i].type, schema[i].bytes); - } + if (isPrimaryKey) { + if (tsCheckTimestamp(pDataBlocks, payloadValues(payload)) != TSDB_CODE_SUCCESS) { + tscInvalidOperationMsg(pInsertParam->msg, "client time/server time can not be mixed up", sToken.z); + return TSDB_CODE_TSC_INVALID_TIME_STAMP; + } + payloadColSetOffset(kvPrimaryKeyStart, colValOffset); + } else { + payloadColSetOffset(kvStart, colValOffset); + if (IS_DATA_COL_ORDERED(spd->orderStatus)) { + kvStart += PAYLOAD_COL_HEAD_LEN; // move to next column } - - ptr += schema[i].bytes; } - - rowSize = (int32_t)(ptr - payload); + + colValOffset += colValAppended; + kvRowLen += kvRowColLen; + dataRowLen += dataRowDeltaColLen; } - *len = rowSize; + if (kvRowLen < dataRowLen) { + payloadSetType(payload, SMEM_ROW_KV); + } else { + payloadSetType(payload, SMEM_ROW_DATA); + } + + *len = (int32_t)(payloadValOffset + colValOffset); + payloadSetTLen(payload, *len); + return TSDB_CODE_SUCCESS; } @@ -536,6 +922,27 @@ static int32_t rowDataCompar(const void *lhs, const void *rhs) { return left > right ? 1 : -1; } } +int32_t schemaIdxCompar(const void *lhs, const void *rhs) { + uint16_t left = *(uint16_t *)lhs; + uint16_t right = *(uint16_t *)rhs; + + if (left == right) { + return 0; + } else { + return left > right ? 1 : -1; + } +} + +int32_t boundIdxCompar(const void *lhs, const void *rhs) { + uint16_t left = *(uint16_t *)POINTER_SHIFT(lhs, sizeof(uint16_t)); + uint16_t right = *(uint16_t *)POINTER_SHIFT(rhs, sizeof(uint16_t)); + + if (left == right) { + return 0; + } else { + return left > right ? 1 : -1; + } +} int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SInsertStatementParam *pInsertParam, int32_t* numOfRows, char *tmpTokenBuf) { @@ -551,21 +958,26 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn int32_t precision = tinfo.precision; + int32_t extendedRowSize = getExtendedRowSize(&tinfo); + + initSMemRowHelper(&pDataBlock->rowHelper, tscGetTableSchema(pDataBlock->pTableMeta), + tscGetNumOfColumns(pDataBlock->pTableMeta), 0); + while (1) { index = 0; sToken = tStrGetToken(*str, &index, false); if (sToken.n == 0 || sToken.type != TK_LP) break; *str += index; - if ((*numOfRows) >= maxRows || pDataBlock->size + tinfo.rowSize >= pDataBlock->nAllocSize) { + if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) { int32_t tSize; - code = tscAllocateMemIfNeed(pDataBlock, tinfo.rowSize, &tSize); + code = tscAllocateMemIfNeed(pDataBlock, extendedRowSize, &tSize); if (code != TSDB_CODE_SUCCESS) { //TODO pass the correct error code to client strcpy(pInsertParam->msg, "client out of memory"); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - ASSERT(tSize > maxRows); + ASSERT(tSize >= maxRows); maxRows = tSize; } @@ -601,9 +1013,10 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols) { pColInfo->numOfCols = numOfCols; pColInfo->numOfBound = numOfCols; - + pColInfo->orderStatus = ORDER_STATUS_ORDERED; pColInfo->boundedColumns = calloc(pColInfo->numOfCols, sizeof(int32_t)); pColInfo->cols = calloc(pColInfo->numOfCols, sizeof(SBoundColumn)); + pColInfo->colIdxInfo = NULL; for (int32_t i = 0; i < pColInfo->numOfCols; ++i) { if (i > 0) { @@ -656,8 +1069,9 @@ int32_t tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_ } } +#if 0 // data block is disordered, sort it in ascending order -void tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf) { +static void tscSortRemoveDataBlockDupRowsOld(STableDataBlocks *dataBuf) { SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; // size is less than the total size, since duplicated rows may be removed yet. @@ -700,12 +1114,91 @@ void tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf) { dataBuf->prevTS = INT64_MIN; } +#endif + +// data block is disordered, sort it in ascending order +int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKeyInfo) { + SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; + int16_t nRows = pBlocks->numOfRows; + + // size is less than the total size, since duplicated rows may be removed yet. + + // if use server time, this block must be ordered + if (dataBuf->tsSource == TSDB_USE_SERVER_TS) { + assert(dataBuf->ordered); + } + // allocate memory + size_t nAlloc = nRows * sizeof(SBlockKeyTuple); + if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { + size_t nRealAlloc = nAlloc + 10 * sizeof(SBlockKeyTuple); + char * tmp = trealloc(pBlkKeyInfo->pKeyTuple, nRealAlloc); + if (tmp == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple *)tmp; + pBlkKeyInfo->maxBytesAlloc = (int32_t)nRealAlloc; + } + memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc); + + SBlockKeyTuple *pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + char * pBlockData = pBlocks->data; + TDRowTLenT totolPayloadTLen = 0; + TDRowTLenT payloadTLen = 0; + int n = 0; + while (n < nRows) { + pBlkKeyTuple->skey = payloadTSKey(pBlockData); + pBlkKeyTuple->payloadAddr = pBlockData; + payloadTLen = payloadTLen(pBlockData); +#if 0 + ASSERT(payloadNCols(pBlockData) <= 4096); + ASSERT(payloadTLen(pBlockData) < 65536); +#endif + totolPayloadTLen += payloadTLen; + // next loop + pBlockData += payloadTLen; + ++pBlkKeyTuple; + ++n; + } + + if (!dataBuf->ordered) { + pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + qsort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataCompar); + + pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + int32_t i = 0; + int32_t j = 1; + while (j < nRows) { + TSKEY ti = (pBlkKeyTuple + i)->skey; + TSKEY tj = (pBlkKeyTuple + j)->skey; + + if (ti == tj) { + totolPayloadTLen -= payloadTLen(pBlkKeyTuple + j); + ++j; + continue; + } + + int32_t nextPos = (++i); + if (nextPos != j) { + memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + j, sizeof(SBlockKeyTuple)); + } + ++j; + } + + dataBuf->ordered = true; + pBlocks->numOfRows = i + 1; + } + + dataBuf->size = sizeof(SSubmitBlk) + totolPayloadTLen; + dataBuf->prevTS = INT64_MIN; + + return 0; +} static int32_t doParseInsertStatement(SInsertStatementParam *pInsertParam, char **str, STableDataBlocks* dataBuf, int32_t *totalNum) { STableComInfo tinfo = tscGetTableInfo(dataBuf->pTableMeta); int32_t maxNumOfRows; - int32_t code = tscAllocateMemIfNeed(dataBuf, tinfo.rowSize, &maxNumOfRows); + int32_t code = tscAllocateMemIfNeed(dataBuf, getExtendedRowSize(&tinfo), &maxNumOfRows); if (TSDB_CODE_SUCCESS != code) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1038,10 +1531,11 @@ static int32_t validateDataSource(SInsertStatementParam *pInsertParam, int32_t t static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDataColInfo* pColInfo, SSchema* pSchema, char* str, char **end) { - pColInfo->numOfBound = 0; + int32_t nCols = pColInfo->numOfCols; - memset(pColInfo->boundedColumns, 0, sizeof(int32_t) * pColInfo->numOfCols); - for(int32_t i = 0; i < pColInfo->numOfCols; ++i) { + pColInfo->numOfBound = 0; + memset(pColInfo->boundedColumns, 0, sizeof(int32_t) * nCols); + for (int32_t i = 0; i < nCols; ++i) { pColInfo->cols[i].hasVal = false; } @@ -1056,6 +1550,8 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat goto _clean; } + bool isOrdered = true; + int32_t lastColIdx = -1; // last column found while (1) { index = 0; sToken = tStrGetToken(str, &index, false); @@ -1076,7 +1572,8 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat bool findColumnIndex = false; // todo speedup by using hash list - for (int32_t t = 0; t < pColInfo->numOfCols; ++t) { + int32_t nScanned = 0, t = lastColIdx + 1; + while (t < nCols) { if (strncmp(sToken.z, pSchema[t].name, sToken.n) == 0 && strlen(pSchema[t].name) == sToken.n) { if (pColInfo->cols[t].hasVal == true) { code = tscInvalidOperationMsg(pInsertParam->msg, "duplicated column name", sToken.z); @@ -1085,10 +1582,39 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat pColInfo->cols[t].hasVal = true; pColInfo->boundedColumns[pColInfo->numOfBound] = t; - pColInfo->numOfBound += 1; + ++pColInfo->numOfBound; findColumnIndex = true; + if (isOrdered && (lastColIdx > t)) { + isOrdered = false; + } + lastColIdx = t; break; } + ++t; + ++nScanned; + } + if (!findColumnIndex) { + t = 0; + int32_t nRemain = nCols - nScanned; + while (t < nRemain) { + if (strncmp(sToken.z, pSchema[t].name, sToken.n) == 0 && strlen(pSchema[t].name) == sToken.n) { + if (pColInfo->cols[t].hasVal == true) { + code = tscInvalidOperationMsg(pInsertParam->msg, "duplicated column name", sToken.z); + goto _clean; + } + + pColInfo->cols[t].hasVal = true; + pColInfo->boundedColumns[pColInfo->numOfBound] = t; + ++pColInfo->numOfBound; + findColumnIndex = true; + if (isOrdered && (lastColIdx > t)) { + isOrdered = false; + } + lastColIdx = t; + break; + } + ++t; + } } if (!findColumnIndex) { @@ -1097,10 +1623,32 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat } } - memset(&pColInfo->boundedColumns[pColInfo->numOfBound], 0 , sizeof(int32_t) * (pColInfo->numOfCols - pColInfo->numOfBound)); + pColInfo->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; + + if (!isOrdered) { + pColInfo->colIdxInfo = tcalloc(pColInfo->numOfBound, sizeof(SBoundIdxInfo)); + if (pColInfo->colIdxInfo == NULL) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _clean; + } + SBoundIdxInfo *pColIdx = pColInfo->colIdxInfo; + for (uint16_t i = 0; i < pColInfo->numOfBound; ++i) { + pColIdx[i].schemaColIdx = (uint16_t)pColInfo->boundedColumns[i]; + pColIdx[i].boundIdx = i; + } + qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); + for (uint16_t i = 0; i < pColInfo->numOfBound; ++i) { + pColIdx[i].finalIdx = i; + } + qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); + } + + memset(&pColInfo->boundedColumns[pColInfo->numOfBound], 0, + sizeof(int32_t) * (pColInfo->numOfCols - pColInfo->numOfBound)); + return TSDB_CODE_SUCCESS; - _clean: +_clean: pInsertParam->sql = NULL; return code; } @@ -1494,21 +2042,24 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow } STableDataBlocks *pTableDataBlock = NULL; - int32_t ret = - tscGetDataBlockFromList(pInsertParam->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), - tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pTableDataBlock, NULL); + int32_t ret = tscGetDataBlockFromList(pInsertParam->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, + sizeof(SSubmitBlk), tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, + &pTableDataBlock, NULL); if (ret != TSDB_CODE_SUCCESS) { pParentSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _error; } - tscAllocateMemIfNeed(pTableDataBlock, tinfo.rowSize, &maxRows); + tscAllocateMemIfNeed(pTableDataBlock, getExtendedRowSize(&tinfo), &maxRows); tokenBuf = calloc(1, TSDB_MAX_BYTES_PER_ROW); if (tokenBuf == NULL) { code = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _error; } + initSMemRowHelper(&pTableDataBlock->rowHelper, tscGetTableSchema(pTableDataBlock->pTableMeta), + tscGetNumOfColumns(pTableDataBlock->pTableMeta), 0); + while ((readLen = tgetline(&line, &n, fp)) != -1) { if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { line[--readLen] = 0; diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index c240dd25d1..f68fc60270 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -291,6 +291,7 @@ static char* normalStmtBuildSql(STscStmt* stmt) { return taosStringBuilderGetResult(&sb, NULL); } +#if 0 static int fillColumnsNull(STableDataBlocks* pBlock, int32_t rowNum) { SParsedDataColInfo* spd = &pBlock->boundColumnInfo; int32_t offset = 0; @@ -318,8 +319,129 @@ static int fillColumnsNull(STableDataBlocks* pBlock, int32_t rowNum) { return TSDB_CODE_SUCCESS; } +#endif + +/** + * input: + * - schema: + * - payload: + * - spd: + * output: + * - pBlock with data block replaced by K-V format + */ +static int refactorPayload(STableDataBlocks* pBlock, int32_t rowNum) { + SParsedDataColInfo* spd = &pBlock->boundColumnInfo; + SSchema* schema = (SSchema*)pBlock->pTableMeta->schema; + SMemRowHelper* pHelper = &pBlock->rowHelper; + STableMeta* pTableMeta = pBlock->pTableMeta; + STableComInfo tinfo = tscGetTableInfo(pTableMeta); + int code = TSDB_CODE_SUCCESS; + int32_t extendedRowSize = getExtendedRowSize(&tinfo); + TDRowTLenT destPayloadSize = sizeof(SSubmitBlk); + + ASSERT(pHelper->allNullLen >= 8); + + TDRowTLenT destAllocSize = sizeof(SSubmitBlk) + rowNum * extendedRowSize; + SSubmitBlk* pDestBlock = tcalloc(destAllocSize, 1); + if (pDestBlock == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memcpy(pDestBlock, pBlock->pData, sizeof(SSubmitBlk)); + char* destPayload = (char*)pDestBlock + sizeof(SSubmitBlk); + + char* srcPayload = (char*)pBlock->pData + sizeof(SSubmitBlk); + + for (int n = 0; n < rowNum; ++n) { + payloadSetNCols(destPayload, spd->numOfBound); + + TDRowTLenT dataRowLen = pHelper->allNullLen; + TDRowTLenT kvRowLen = TD_MEM_ROW_KV_VER_SIZE + sizeof(SColIdx) * spd->numOfBound; + TDRowTLenT payloadValOffset = payloadValuesOffset(destPayload); // rely on payloadNCols + TDRowLenT colValOffset = 0; + + char* kvPrimaryKeyStart = destPayload + PAYLOAD_HEADER_LEN; // primaryKey in 1st column tuple + char* kvStart = kvPrimaryKeyStart + PAYLOAD_COL_HEAD_LEN; // the column tuple behind the primaryKey + + for (int32_t i = 0; i < spd->numOfBound; ++i) { + int32_t colIndex = spd->boundedColumns[i]; + ASSERT(spd->cols[colIndex].hasVal); + char* start = srcPayload + spd->cols[colIndex].offset; + SSchema* pSchema = &schema[colIndex]; // get colId here + bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); + + // the primary key locates in 1st column + if (!IS_DATA_COL_ORDERED(spd->orderStatus)) { + ASSERT(spd->colIdxInfo != NULL); + if (!isPrimaryKey) { + kvStart = POINTER_SHIFT(kvPrimaryKeyStart, spd->colIdxInfo[i].finalIdx * PAYLOAD_COL_HEAD_LEN); + } else { + ASSERT(spd->colIdxInfo[i].finalIdx == 0); + } + } + if (isPrimaryKey) { + payloadColSetId(kvPrimaryKeyStart, pSchema->colId); + payloadColSetType(kvPrimaryKeyStart, pSchema->type); + payloadColSetOffset(kvPrimaryKeyStart, colValOffset); + memcpy(POINTER_SHIFT(destPayload, payloadValOffset + colValOffset), start, TYPE_BYTES[pSchema->type]); + colValOffset += TYPE_BYTES[pSchema->type]; + kvRowLen += TYPE_BYTES[pSchema->type]; + } else { + payloadColSetId(kvStart, pSchema->colId); + payloadColSetType(kvStart, pSchema->type); + payloadColSetOffset(kvStart, colValOffset); + if (IS_VAR_DATA_TYPE(pSchema->type)) { + varDataCopy(POINTER_SHIFT(destPayload, payloadValOffset + colValOffset), start); + colValOffset += varDataTLen(start); + kvRowLen += varDataTLen(start); + if (pSchema->type == TSDB_DATA_TYPE_BINARY) { + dataRowLen += (varDataLen(start) - CHAR_BYTES); + } else if (pSchema->type == TSDB_DATA_TYPE_NCHAR) { + dataRowLen += (varDataLen(start) - TSDB_NCHAR_SIZE); + } else { + ASSERT(0); + } + } else { + memcpy(POINTER_SHIFT(destPayload, payloadValOffset + colValOffset), start, TYPE_BYTES[pSchema->type]); + colValOffset += TYPE_BYTES[pSchema->type]; + kvRowLen += TYPE_BYTES[pSchema->type]; + } + + if (IS_DATA_COL_ORDERED(spd->orderStatus)) { + kvStart += PAYLOAD_COL_HEAD_LEN; // move to next column + } + } + } // end of column + + if (kvRowLen < dataRowLen) { + payloadSetType(destPayload, SMEM_ROW_KV); + } else { + payloadSetType(destPayload, SMEM_ROW_DATA); + } + + ASSERT(colValOffset <= TSDB_MAX_BYTES_PER_ROW); + + TDRowTLenT len = payloadValOffset + colValOffset; + payloadSetTLen(destPayload, len); + + // next loop + srcPayload += pBlock->rowSize; + destPayload += len; + + destPayloadSize += len; + } // end of row + + ASSERT(destPayloadSize <= destAllocSize); + + tfree(pBlock->pData); + pBlock->pData = (char*)pDestBlock; + pBlock->nAllocSize = destAllocSize; + pBlock->size = destPayloadSize; + + return code; +} +#if 0 int32_t fillTablesColumnsNull(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; @@ -342,17 +464,98 @@ int32_t fillTablesColumnsNull(SSqlObj* pSql) { return TSDB_CODE_SUCCESS; } +#endif - - -//////////////////////////////////////////////////////////////////////////////// -// functions for insertion statement preparation -static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, TAOS_BIND* bind, int32_t colNum) { - if (bind->is_null != NULL && *(bind->is_null)) { - setNull(data + param->offset, param->type, param->bytes); +/** + * check and sort + */ +static int initPayloadEnv(STableDataBlocks* pBlock, int32_t rowNum) { + SParsedDataColInfo* spd = &pBlock->boundColumnInfo; + if (spd->orderStatus != ORDER_STATUS_UNKNOWN) { return TSDB_CODE_SUCCESS; } + bool isOrdered = true; + int32_t lastColIdx = -1; + for (int32_t i = 0; i < spd->numOfBound; ++i) { + ASSERT(spd->cols[i].hasVal); + int32_t colIdx = spd->boundedColumns[i]; + if (isOrdered) { + if (lastColIdx > colIdx) { + isOrdered = false; + break; + } else { + lastColIdx = colIdx; + } + } + } + + spd->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; + + if (isOrdered) { + spd->colIdxInfo = NULL; + } else { + spd->colIdxInfo = calloc(spd->numOfBound, sizeof(SBoundIdxInfo)); + if (spd->colIdxInfo == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + SBoundIdxInfo* pColIdx = spd->colIdxInfo; + for (uint16_t i = 0; i < spd->numOfBound; ++i) { + pColIdx[i].schemaColIdx = (uint16_t)spd->boundedColumns[i]; + pColIdx[i].boundIdx = i; + } + qsort(pColIdx, spd->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); + for (uint16_t i = 0; i < spd->numOfBound; ++i) { + pColIdx[i].finalIdx = i; + } + qsort(pColIdx, spd->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); + } + + return TSDB_CODE_SUCCESS; +} + +/** + * Refactor the raw payload structure to K-V format as the in tsParseOneRow() + */ +int32_t fillTablesPayload(SSqlObj* pSql) { + SSqlCmd* pCmd = &pSql->cmd; + int code = TSDB_CODE_SUCCESS; + + STableDataBlocks** p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, NULL); + + STableDataBlocks* pOneTableBlock = *p; + while (pOneTableBlock) { + SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData; + + if (pBlocks->numOfRows > 0) { + initSMemRowHelper(&pOneTableBlock->rowHelper, tscGetTableSchema(pOneTableBlock->pTableMeta), + tscGetNumOfColumns(pOneTableBlock->pTableMeta), 0); + if ((code = initPayloadEnv(pOneTableBlock, pBlocks->numOfRows)) != TSDB_CODE_SUCCESS) { + return code; + } + if ((code = refactorPayload(pOneTableBlock, pBlocks->numOfRows)) != TSDB_CODE_SUCCESS) { + return code; + }; + } + + p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, p); + if (p == NULL) { + break; + } + + pOneTableBlock = *p; + } + + return code; +} + //////////////////////////////////////////////////////////////////////////////// + // functions for insertion statement preparation + static int doBindParam(STableDataBlocks* pBlock, char* data, SParamInfo* param, TAOS_BIND* bind, int32_t colNum) { + if (bind->is_null != NULL && *(bind->is_null)) { + setNull(data + param->offset, param->type, param->bytes); + return TSDB_CODE_SUCCESS; + } + #if 0 if (0) { // allow user bind param data with different type @@ -1106,9 +1309,12 @@ static int insertStmtExecute(STscStmt* stmt) { pBlk->uid = pTableMeta->id.uid; pBlk->tid = pTableMeta->id.tid; - fillTablesColumnsNull(stmt->pSql); + int code = fillTablesPayload(stmt->pSql); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - int code = tscMergeTableDataBlocks(&stmt->pSql->cmd.insertParam, false); + code = tscMergeTableDataBlocks(&stmt->pSql->cmd.insertParam, false); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1185,7 +1391,7 @@ static int insertBatchStmtExecute(STscStmt* pStmt) { return TSDB_CODE_TSC_APP_ERROR; } - fillTablesColumnsNull(pStmt->pSql); + fillTablesPayload(pStmt->pSql); if ((code = tscMergeTableDataBlocks(&pStmt->pSql->cmd.insertParam, false)) != TSDB_CODE_SUCCESS) { return code; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 3166b47ae8..af57719ccf 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1396,12 +1396,16 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { const char* msg4 = "invalid data type"; const char* msg5 = "invalid binary/nchar column length"; const char* msg6 = "invalid column name"; + const char* msg7 = "too many columns"; // number of fields no less than 2 size_t numOfCols = taosArrayGetSize(pFieldList); - if (numOfCols <= 1 || numOfCols > TSDB_MAX_COLUMNS) { + if (numOfCols <= 1 ) { invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); return false; + } else if (numOfCols > TSDB_MAX_COLUMNS) { + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); + return false; } // first column must be timestamp @@ -1536,13 +1540,20 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { const char* msg4 = "invalid tag name"; const char* msg5 = "invalid binary/nchar tag length"; const char* msg6 = "invalid data type in tags"; + const char* msg7 = "too many columns"; STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; int32_t numOfTags = tscGetNumOfTags(pTableMeta); int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - + + // no more max columns + if (numOfTags + numOfCols >= TSDB_MAX_COLUMNS) { + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); + return false; + } + // no more than 6 tags if (numOfTags == TSDB_MAX_TAGS) { char msg[128] = {0}; @@ -3106,7 +3117,7 @@ int32_t doGetColumnIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColum if (isTablenameToken(pToken)) { pIndex->columnIndex = TSDB_TBNAME_COLUMN_INDEX; - } else if (strlen(DEFAULT_PRIMARY_TIMESTAMP_COL_NAME) == pToken->n && + } else if (strlen(DEFAULT_PRIMARY_TIMESTAMP_COL_NAME) == pToken->n && strncasecmp(pToken->z, DEFAULT_PRIMARY_TIMESTAMP_COL_NAME, pToken->n) == 0) { pIndex->columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX; // just make runtime happy, need fix java test case InsertSpecialCharacterJniTest } else if (pToken->n == 0) { @@ -3708,9 +3719,9 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, * make memory sanitizer happy; */ if (pRight->value.nLen == 0) { - bufLen = pRight->value.nLen + 2; + bufLen = pRight->value.nLen + 2; } else { - bufLen = pRight->value.nLen + 1; + bufLen = pRight->value.nLen + 1; } } @@ -5150,7 +5161,7 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq int32_t type = 0; if ((ret = getQueryCondExpr(&pSql->cmd, pQueryInfo, pExpr, &condExpr, &type, (*pExpr)->tokenId)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; + goto PARSE_WHERE_EXIT; } tSqlExprCompact(pExpr); @@ -5160,7 +5171,7 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq // 1. check if it is a join query if ((ret = validateJoinExpr(&pSql->cmd, pQueryInfo, &condExpr)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; + goto PARSE_WHERE_EXIT; } // 2. get the query time range @@ -5999,6 +6010,16 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidOperationMsg(pMsg, msg22); } + SSchema* pSchema = (SSchema*) pTableMetaInfo->pTableMeta->schema; + int16_t numOfColumns = pTableMetaInfo->pTableMeta->tableInfo.numOfColumns; + int16_t i; + uint32_t nLen = 0; + for (i = 0; i < numOfColumns; ++i) { + nLen += pSchema[i].colId != columnIndex.columnIndex ? pSchema[i].bytes : pItem->bytes; + } + if (nLen >= TSDB_MAX_BYTES_PER_ROW) { + return invalidOperationMsg(pMsg, msg24); + } TAOS_FIELD f = tscCreateField(pColSchema->type, name.z, pItem->bytes); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); }else if (pAlterSQL->type == TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN) { @@ -6040,6 +6061,17 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidOperationMsg(pMsg, msg22); } + SSchema* pSchema = (SSchema*) pTableMetaInfo->pTableMeta->schema; + int16_t numOfColumns = pTableMetaInfo->pTableMeta->tableInfo.numOfColumns; + int16_t i; + uint32_t nLen = 0; + for (i = 0; i < numOfColumns; ++i) { + nLen += pSchema[i].colId != columnIndex.columnIndex ? pSchema[i].bytes : pItem->bytes; + } + if (nLen >= TSDB_MAX_BYTES_PER_ROW) { + return invalidOperationMsg(pMsg, msg24); + } + TAOS_FIELD f = tscCreateField(pColSchema->type, name.z, pItem->bytes); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } @@ -8048,8 +8080,12 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { uint32_t maxSize = tscGetTableMetaMaxSize(); char name[TSDB_TABLE_FNAME_LEN] = {0}; - char buf[80 * 1024] = {0}; - assert(maxSize < 80 * 1024); + assert(maxSize < 80 * TSDB_MAX_COLUMNS); + if (!pSql->pBuf) { + if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } pTableMeta = calloc(1, maxSize); plist = taosArrayInit(4, POINTER_BYTES); @@ -8066,7 +8102,7 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pTableMeta->id.uid > 0) { if (pTableMeta->tableType == TSDB_CHILD_TABLE) { - code = tscCreateTableMetaFromSTableMeta(pTableMeta, name, buf); + code = tscCreateTableMetaFromSTableMeta(pTableMeta, name, pSql->pBuf); // create the child table meta from super table failed, try load it from mnode if (code != TSDB_CODE_SUCCESS) { @@ -8089,7 +8125,7 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { // avoid mem leak, may should update pTableMeta const char* px = tNameGetTableName(pname); if (taosHashGet(pCmd->pTableMetaMap, px, strlen(px)) == NULL) { - STableMeta* pMeta = tscTableMetaDup(pTableMeta); + STableMeta* pMeta = tscTableMetaDup(pTableMeta); STableMetaVgroupInfo p = { .pTableMeta = pMeta, .pVgroupInfo = NULL}; taosHashPut(pCmd->pTableMetaMap, px, strlen(px), &p, sizeof(STableMetaVgroupInfo)); } @@ -8133,7 +8169,7 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { // load the table meta for a given table name list if (taosArrayGetSize(plist) > 0 || taosArrayGetSize(pVgroupList) > 0 || (pQueryInfo->pUdfInfo && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0)) { - code = getMultiTableMetaFromMnode(pSql, plist, pVgroupList, pQueryInfo->pUdfInfo, tscTableMetaCallBack); + code = getMultiTableMetaFromMnode(pSql, plist, pVgroupList, pQueryInfo->pUdfInfo, tscTableMetaCallBack, true); } _end: @@ -8262,7 +8298,9 @@ static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pS // union all is not support currently SSqlNode* p = taosArrayGetP(subInfo->pSubquery, 0); - + if (taosArrayGetSize(subInfo->pSubquery) >= 2) { + return invalidOperationMsg(msgBuf, "not support union in subquery"); + } SQueryInfo* pSub = calloc(1, sizeof(SQueryInfo)); tscInitQueryInfo(pSub); @@ -8630,7 +8668,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf } taosArrayAddBatch(pQueryInfo->exprList1, (void*) p, numOfExpr); - tfree(p); + tfree(p); } #if 0 diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 4abea9fed8..e975dd7b06 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1955,7 +1955,6 @@ static int32_t tableMetaMsgConvert(STableMetaMsg* pMetaMsg) { pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); pMetaMsg->uid = htobe64(pMetaMsg->uid); -// pMetaMsg->contLen = htonl(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && @@ -2020,13 +2019,14 @@ static void doAddTableMetaToLocalBuf(STableMeta* pTableMeta, STableMetaMsg* pMet int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); // The super tableMeta already exists, create it according to tableMeta and add it to hash map - STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg); + if (updateSTable) { + STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg); + uint32_t size = tscGetTableMetaSize(pSupTableMeta); + int32_t code = taosHashPut(tscTableMetaInfo, pTableMeta->sTableName, len, pSupTableMeta, size); + assert(code == TSDB_CODE_SUCCESS); - uint32_t size = tscGetTableMetaSize(pSupTableMeta); - int32_t code = taosHashPut(tscTableMetaInfo, pTableMeta->sTableName, len, pSupTableMeta, size); - assert(code == TSDB_CODE_SUCCESS); - - tfree(pSupTableMeta); + tfree(pSupTableMeta); + } CChildTableMeta* cMeta = tscCreateChildMeta(pTableMeta); taosHashPut(tscTableMetaInfo, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), cMeta, sizeof(CChildTableMeta)); @@ -2183,10 +2183,10 @@ int tscProcessMultiTableMetaRsp(SSqlObj *pSql) { } SSqlCmd *pParentCmd = &pParentSql->cmd; - SHashObj *pSet = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + SHashObj *pSet = taosHashInit(pMultiMeta->numOfVgroup, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); - char* pMsg = pMultiMeta->meta; char* buf = NULL; + char* pMsg = pMultiMeta->meta; if (pMultiMeta->compressed) { buf = malloc(pMultiMeta->rawLen - sizeof(SMultiTableMeta)); int32_t len = tsDecompressString(pMultiMeta->meta, pMultiMeta->contLen - sizeof(SMultiTableMeta), 1, @@ -2207,6 +2207,7 @@ int tscProcessMultiTableMetaRsp(SSqlObj *pSql) { return code; } + bool freeMeta = false; STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, pMetaMsg->tableFname); @@ -2221,22 +2222,27 @@ int tscProcessMultiTableMetaRsp(SSqlObj *pSql) { SName sn = {0}; tNameFromString(&sn, pMetaMsg->tableFname, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - const char* tableName = tNameGetTableName(&sn); - size_t keyLen = strlen(tableName); + if (pMultiMeta->metaClone == 1 || pTableMeta->tableType == TSDB_SUPER_TABLE) { + STableMetaVgroupInfo p = {.pTableMeta = pTableMeta,}; - STableMetaVgroupInfo p = {.pTableMeta = pTableMeta,}; - taosHashPut(pParentCmd->pTableMetaMap, tableName, keyLen, &p, sizeof(STableMetaVgroupInfo)); + const char* tableName = tNameGetTableName(&sn); + size_t keyLen = strlen(tableName); + taosHashPut(pParentCmd->pTableMetaMap, tableName, keyLen, &p, sizeof(STableMetaVgroupInfo)); + } else { + freeMeta = true; + } - bool addToBuf = false; - if (taosHashGet(pSet, &pMetaMsg->uid, sizeof(pMetaMsg->uid)) == NULL) { - addToBuf = true; - taosHashPut(pSet, &pMetaMsg->uid, sizeof(pMetaMsg->uid), "", 0); + // for each super table, only update meta information once + bool updateStableMeta = false; + if (pTableMeta->tableType == TSDB_CHILD_TABLE && taosHashGet(pSet, &pMetaMsg->suid, sizeof(pMetaMsg->suid)) == NULL) { + updateStableMeta = true; + taosHashPut(pSet, &pTableMeta->suid, sizeof(pMetaMsg->suid), "", 0); } // create the tableMeta and add it into the TableMeta map - doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, addToBuf); + doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, updateStableMeta); - // if the vgroup is not updated in current process, update it. + // for each vgroup, only update the information once. int64_t vgId = pMetaMsg->vgroup.vgId; if (pTableMeta->tableType != TSDB_SUPER_TABLE && taosHashGet(pSet, &vgId, sizeof(vgId)) == NULL) { doUpdateVgroupInfo(pTableMeta, &pMetaMsg->vgroup); @@ -2244,6 +2250,9 @@ int tscProcessMultiTableMetaRsp(SSqlObj *pSql) { } pMsg += pMetaMsg->contLen; + if (freeMeta) { + tfree(pTableMeta); + } } for(int32_t i = 0; i < pMultiMeta->numOfVgroup; ++i) { @@ -2683,7 +2692,7 @@ static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaIn return code; } -int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp) { +int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp, bool metaClone) { SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); if (NULL == pNew) { tscError("0x%"PRIx64" failed to allocate sqlobj to get multiple table meta", pSql->self); @@ -2706,6 +2715,7 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg } SMultiTableInfoMsg* pInfo = (SMultiTableInfoMsg*) pNew->cmd.payload; + pInfo->metaClone = metaClone? 1:0; pInfo->numOfTables = htonl((uint32_t) taosArrayGetSize(pNameList)); pInfo->numOfVgroups = htonl((uint32_t) taosArrayGetSize(pVgroupNameList)); pInfo->numOfUdfs = htonl(numOfUdf); @@ -2794,22 +2804,24 @@ int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool taosHashGetClone(tscTableMetaInfo, name, len, NULL, pTableMetaInfo->pTableMeta, -1); // TODO resize the tableMeta - char buf[80*1024] = {0}; - assert(size < 80*1024); + assert(size < 80 * TSDB_MAX_COLUMNS); + if (!pSql->pBuf) { + if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } STableMeta* pMeta = pTableMetaInfo->pTableMeta; if (pMeta->id.uid > 0) { // in case of child table, here only get the if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, buf); + int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, pSql->pBuf); if (code != TSDB_CODE_SUCCESS) { return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); } } - return TSDB_CODE_SUCCESS; } - return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); } @@ -2924,7 +2936,6 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) { if (allVgroupInfoRetrieved(pQueryInfo)) { return TSDB_CODE_SUCCESS; } - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 9250a1bbc3..7675bded65 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -945,28 +945,35 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { return TSDB_CODE_TSC_DISCONNECTED; } - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - pSql->pTscObj = taos; - pSql->signature = pSql; - pSql->cmd.pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - int32_t length = (int32_t)strlen(tableNameList); + if (length == 0) { + return TSDB_CODE_SUCCESS; + } + if (length > MAX_TABLE_NAME_LENGTH) { - tscError("0x%"PRIx64" tableNameList too long, length:%d, maximum allowed:%d", pSql->self, length, MAX_TABLE_NAME_LENGTH); - tscFreeSqlObj(pSql); + tscError("tableNameList too long, length:%d, maximum allowed:%d", length, MAX_TABLE_NAME_LENGTH); return TSDB_CODE_TSC_INVALID_OPERATION; } char *str = calloc(1, length + 1); if (str == NULL) { - tscError("0x%"PRIx64" failed to allocate sql string buffer", pSql->self); - tscFreeSqlObj(pSql); + tscError("failed to allocate sql string buffer, size:%d", length); return TSDB_CODE_TSC_OUT_OF_MEMORY; } strtolower(str, tableNameList); SArray* plist = taosArrayInit(4, POINTER_BYTES); SArray* vgroupList = taosArrayInit(4, POINTER_BYTES); + if (plist == NULL || vgroupList == NULL) { + tfree(str); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + tscAllocPayload(&pSql->cmd, 1024); + + pSql->pTscObj = taos; + pSql->signature = pSql; int32_t code = (uint8_t) tscTransferTableNameList(pSql, str, length, plist); free(str); @@ -976,10 +983,11 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { return code; } + pSql->cmd.pTableMetaMap = taosHashInit(taosArrayGetSize(plist), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); registerSqlObj(pSql); tscDebug("0x%"PRIx64" load multiple table meta, tableNameList: %s pObj:%p", pSql->self, tableNameList, pObj); - code = getMultiTableMetaFromMnode(pSql, plist, vgroupList, NULL, loadMultiTableMetaCallback); + code = getMultiTableMetaFromMnode(pSql, plist, vgroupList, NULL, loadMultiTableMetaCallback, false); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { code = TSDB_CODE_SUCCESS; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 4a9ceeca70..b72bd78b1b 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -712,6 +712,15 @@ static void updateQueryTimeRange(SQueryInfo* pQueryInfo, STimeWindow* win) { pQueryInfo->window = *win; } +int32_t tagValCompar(const void* p1, const void* p2) { + const STidTags* t1 = (const STidTags*) varDataVal(p1); + const STidTags* t2 = (const STidTags*) varDataVal(p2); + + __compar_fn_t func = getComparFunc(t1->padding, 0); + + return func(t1->tag, t2->tag); +} + int32_t tidTagsCompar(const void* p1, const void* p2) { const STidTags* t1 = (const STidTags*) (p1); const STidTags* t2 = (const STidTags*) (p2); @@ -720,28 +729,7 @@ int32_t tidTagsCompar(const void* p1, const void* p2) { return (t1->vgId > t2->vgId) ? 1 : -1; } - tstr* tag1 = (tstr*) t1->tag; - tstr* tag2 = (tstr*) t2->tag; - - if (tag1->len != tag2->len) { - return (tag1->len > tag2->len)? 1: -1; - } - - return strncmp(tag1->data, tag2->data, tag1->len); -} - -int32_t tagValCompar(const void* p1, const void* p2) { - const STidTags* t1 = (const STidTags*) varDataVal(p1); - const STidTags* t2 = (const STidTags*) varDataVal(p2); - - tstr* tag1 = (tstr*) t1->tag; - tstr* tag2 = (tstr*) t2->tag; - - if (tag1->len != tag2->len) { - return (tag1->len > tag2->len)? 1: -1; - } - - return memcmp(tag1->data, tag2->data, tag1->len); + return tagValCompar(p1, p2); } void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables) { @@ -871,6 +859,12 @@ static bool checkForDuplicateTagVal(SSchema* pColSchema, SJoinSupporter* p1, SSq return true; } +static void setTidTagType(SJoinSupporter* p, uint8_t type) { + for (int32_t i = 0; i < p->num; ++i) { + STidTags * tag = (STidTags*) varDataVal(p->pIdTagList + i * p->tagSize); + tag->padding = type; + } +} static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray* resList) { int16_t joinNum = pParentSql->subState.numOfSub; @@ -890,6 +884,8 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar for (int32_t i = 0; i < joinNum; i++) { SJoinSupporter* p = pParentSql->pSubs[i]->param; + setTidTagType(p, pColSchema->type); + ctxlist[i].p = p; ctxlist[i].res = taosArrayInit(p->num, size); @@ -2296,6 +2292,7 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { SArray* pColList = pNewQueryInfo->colList; pNewQueryInfo->colList = NULL; + pNewQueryInfo->fillType = TSDB_FILL_NONE; tscClearSubqueryInfo(pCmd); tscFreeSqlResult(pSql); @@ -2439,9 +2436,9 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { tOrderDescriptor *pDesc = NULL; pRes->qId = 0x1; // hack the qhandle check - - const uint32_t nBufferSize = (1u << 16u); // 64KB - + + const uint32_t nBufferSize = (1u << 18u); // 256KB + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SSubqueryState *pState = &pSql->subState; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index e4ddff1c90..6cb8e9a0de 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1488,6 +1488,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { pSql->signature = NULL; pSql->fp = NULL; tfree(pSql->sqlstr); + tfree(pSql->pBuf); tfree(pSql->pSubs); pSql->subState.numOfSub = 0; @@ -1499,7 +1500,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { memset(pCmd->payload, 0, (size_t)pCmd->allocSize); tfree(pCmd->payload); pCmd->allocSize = 0; - + tsem_destroy(&pSql->rspSem); memset(pSql, 0, sizeof(*pSql)); free(pSql); @@ -1508,6 +1509,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo) { tfree(pColInfo->boundedColumns); tfree(pColInfo->cols); + tfree(pColInfo->colIdxInfo); } void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) { @@ -1762,11 +1764,107 @@ int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, i return TSDB_CODE_SUCCESS; } -static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bool includeSchema) { +static SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { + SSchema* pSchema = pBuilder->pSchema; + char* p = (char*)pBuilder->buf; + int toffset = 0; + uint16_t nCols = pBuilder->nCols; + + uint8_t memRowType = payloadType(p); + uint16_t nColsBound = payloadNCols(p); + if (pBuilder->nCols <= 0 || nColsBound <= 0) { + return NULL; + } + char* pVals = POINTER_SHIFT(p, payloadValuesOffset(p)); + SMemRow* memRow = (SMemRow)pBuilder->pDataBlock; + memRowSetType(memRow, memRowType); + + // ----------------- Raw payload structure for row: + /* |<------------ Head ------------->|<----------- body of column data tuple ------------------->| + * | |<----------------- flen ------------->|<--- value part --->| + * |SMemRowType| dataTLen | nCols | colId | colType | offset | ... | value |...|...|... | + * +-----------+----------+----------+--------------------------------------|--------------------| + * | uint8_t | uint32_t | uint16_t | int16_t | uint8_t | uint16_t | ... |.......|...|...|... | + * +-----------+----------+----------+--------------------------------------+--------------------| + * 1. offset in column data tuple starts from the value part in case of uint16_t overflow. + * 2. dataTLen: total length including the header and body. + */ + + if (memRowType == SMEM_ROW_DATA) { + SDataRow trow = (SDataRow)memRowDataBody(memRow); + dataRowSetLen(trow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pBuilder->flen)); + dataRowSetVersion(trow, pBuilder->sversion); + + p = (char*)payloadBody(pBuilder->buf); + uint16_t i = 0, j = 0; + while (j < nCols) { + if (i >= nColsBound) { + break; + } + int16_t colId = payloadColId(p); + if (colId == pSchema[j].colId) { + // ASSERT(payloadColType(p) == pSchema[j].type); + tdAppendColVal(trow, POINTER_SHIFT(pVals, payloadColOffset(p)), pSchema[j].type, toffset); + toffset += TYPE_BYTES[pSchema[j].type]; + p = payloadNextCol(p); + ++i; + ++j; + } else if (colId < pSchema[j].colId) { + p = payloadNextCol(p); + ++i; + } else { + tdAppendColVal(trow, tdGetNullVal(pSchema[j].type), pSchema[j].type, toffset); + toffset += TYPE_BYTES[pSchema[j].type]; + ++j; + } + } + + while (j < nCols) { + tdAppendColVal(trow, tdGetNullVal(pSchema[j].type), pSchema[j].type, toffset); + toffset += TYPE_BYTES[pSchema[j].type]; + ++j; + } + + #if 0 // no need anymore + while (i < nColsBound) { + p = payloadNextCol(p); + ++i; + } + #endif + + } else if (memRowType == SMEM_ROW_KV) { + SKVRow kvRow = (SKVRow)memRowKvBody(memRow); + kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nColsBound)); + kvRowSetNCols(kvRow, nColsBound); + memRowSetKvVersion(memRow, pBuilder->sversion); + + p = (char*)payloadBody(pBuilder->buf); + int i = 0; + while (i < nColsBound) { + int16_t colId = payloadColId(p); + uint8_t colType = payloadColType(p); + tdAppendKvColVal(kvRow, POINTER_SHIFT(pVals,payloadColOffset(p)), colId, colType, toffset); + toffset += sizeof(SColIdx); + p = payloadNextCol(p); + ++i; + } + + } else { + ASSERT(0); + } + int32_t rowTLen = memRowTLen(memRow); + pBuilder->pDataBlock = (char*)pBuilder->pDataBlock + rowTLen; // next row + pBuilder->pSubmitBlk->dataLen += rowTLen; + + return memRow; +} + +// Erase the empty space reserved for binary data +static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bool includeSchema, SBlockKeyTuple *blkKeyTuple) { // TODO: optimize this function, handle the case while binary is not presented - STableMeta* pTableMeta = pTableDataBlock->pTableMeta; - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - SSchema* pSchema = tscGetTableSchema(pTableMeta); + STableMeta* pTableMeta = pTableDataBlock->pTableMeta; + STableComInfo tinfo = tscGetTableInfo(pTableMeta); + SSchema* pSchema = tscGetTableSchema(pTableMeta); SSubmitBlk* pBlock = pDataBlock; memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk)); @@ -1801,21 +1899,19 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); pBlock->dataLen = 0; int32_t numOfRows = htons(pBlock->numOfRows); - + + SMemRowBuilder rowBuilder; + rowBuilder.pSchema = pSchema; + rowBuilder.sversion = pTableMeta->sversion; + rowBuilder.flen = flen; + rowBuilder.nCols = tinfo.numOfColumns; + rowBuilder.pDataBlock = pDataBlock; + rowBuilder.pSubmitBlk = pBlock; + rowBuilder.buf = p; + for (int32_t i = 0; i < numOfRows; ++i) { - SDataRow trow = (SDataRow) pDataBlock; - dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + flen)); - dataRowSetVersion(trow, pTableMeta->sversion); - - int toffset = 0; - for (int32_t j = 0; j < tinfo.numOfColumns; j++) { - tdAppendColVal(trow, p, pSchema[j].type, pSchema[j].bytes, toffset); - toffset += TYPE_BYTES[pSchema[j].type]; - p += pSchema[j].bytes; - } - - pDataBlock = (char*)pDataBlock + dataRowLen(trow); - pBlock->dataLen += dataRowLen(trow); + rowBuilder.buf = (blkKeyTuple + i)->payloadAddr; + tdGenMemRowFromBuilder(&rowBuilder); } int32_t len = pBlock->dataLen + pBlock->schemaLen; @@ -1826,7 +1922,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo } static int32_t getRowExpandSize(STableMeta* pTableMeta) { - int32_t result = TD_DATA_ROW_HEAD_SIZE; + int32_t result = TD_MEM_ROW_DATA_HEAD_SIZE; int32_t columns = tscGetNumOfColumns(pTableMeta); SSchema* pSchema = tscGetTableSchema(pTableMeta); for(int32_t i = 0; i < columns; i++) { @@ -1862,13 +1958,16 @@ static void extractTableNameList(SInsertStatementParam *pInsertParam, bool freeB int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBlockMap) { const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); - - void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); + int code = 0; + void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); STableDataBlocks** p = taosHashIterate(pInsertParam->pTableBlockHashList, NULL); STableDataBlocks* pOneTableBlock = *p; + + SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock + while(pOneTableBlock) { SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; if (pBlocks->numOfRows > 0) { @@ -1882,6 +1981,7 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl tscError("0x%"PRIx64" failed to prepare the data block buffer for merging table data, code:%d", pInsertParam->objectId, ret); taosHashCleanup(pVnodeDataBlockHashList); tscDestroyBlockArrayList(pVnodeDataBlockList); + tfree(blkKeyInfo.pKeyTuple); return ret; } @@ -1902,16 +2002,26 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl taosHashCleanup(pVnodeDataBlockHashList); tscDestroyBlockArrayList(pVnodeDataBlockList); tfree(dataBuf->pData); + tfree(blkKeyInfo.pKeyTuple); return TSDB_CODE_TSC_OUT_OF_MEMORY; } } - tscSortRemoveDataBlockDupRows(pOneTableBlock); - char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); + if ((code = tscSortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0) { + taosHashCleanup(pVnodeDataBlockHashList); + tscDestroyBlockArrayList(pVnodeDataBlockList); + tfree(dataBuf->pData); + tfree(blkKeyInfo.pKeyTuple); + return code; + } - tscDebug("0x%"PRIx64" name:%s, tid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pInsertParam->objectId, tNameGetTableName(&pOneTableBlock->tableName), - pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey)); + ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); + + SBlockKeyTuple* pLastKeyTuple = blkKeyInfo.pKeyTuple + pBlocks->numOfRows - 1; + tscDebug("0x%" PRIx64 " name:%s, tid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, + pInsertParam->objectId, tNameGetTableName(&pOneTableBlock->tableName), pBlocks->tid, pBlocks->numOfRows, + pBlocks->sversion, blkKeyInfo.pKeyTuple->skey, pLastKeyTuple->skey); int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + expandSize) + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); @@ -1922,7 +2032,7 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl pBlocks->schemaLen = 0; // erase the empty space reserved for binary data - int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, pInsertParam->schemaAttached); + int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, pInsertParam->schemaAttached, blkKeyInfo.pKeyTuple); assert(finalLen <= len); dataBuf->size += (finalLen + sizeof(SSubmitBlk)); @@ -1950,6 +2060,7 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl // free the table data blocks; pInsertParam->pDataBlocks = pVnodeDataBlockList; taosHashCleanup(pVnodeDataBlockHashList); + tfree(blkKeyInfo.pKeyTuple); return TSDB_CODE_SUCCESS; } @@ -4574,8 +4685,13 @@ static int32_t doAddTableName(char* nextStr, char** str, SArray* pNameArray, SSq strncpy(tablename, *str, TSDB_TABLE_FNAME_LEN); len = (int32_t) strlen(tablename); } else { - memcpy(tablename, *str, nextStr - (*str)); len = (int32_t)(nextStr - (*str)); + if (len >= TSDB_TABLE_NAME_LEN) { + sprintf(pCmd->payload, "table name too long"); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + memcpy(tablename, *str, nextStr - (*str)); tablename[len] = '\0'; } @@ -4587,9 +4703,8 @@ static int32_t doAddTableName(char* nextStr, char** str, SArray* pNameArray, SSq // Check if the table name available or not if (tscValidateName(&sToken) != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; sprintf(pCmd->payload, "table name is invalid"); - return code; + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } SName name = {0}; @@ -4605,6 +4720,15 @@ static int32_t doAddTableName(char* nextStr, char** str, SArray* pNameArray, SSq return TSDB_CODE_SUCCESS; } +int32_t nameComparFn(const void* n1, const void* n2) { + int32_t ret = strcmp(*(char**)n1, *(char**)n2); + if (ret == 0) { + return 0; + } else { + return ret > 0? 1:-1; + } +} + int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t length, SArray* pNameArray) { SSqlCmd *pCmd = &pSql->cmd; @@ -4640,12 +4764,44 @@ int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t lengt } } - if (taosArrayGetSize(pNameArray) > TSDB_MULTI_TABLEMETA_MAX_NUM) { + size_t len = taosArrayGetSize(pNameArray); + if (len == 1) { + return TSDB_CODE_SUCCESS; + } + + if (len > TSDB_MULTI_TABLEMETA_MAX_NUM) { code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; sprintf(pCmd->payload, "tables over the max number"); return code; } + taosArraySort(pNameArray, nameComparFn); + + int32_t pos = 0; + for(int32_t i = 1; i < len; ++i) { + char** p1 = taosArrayGet(pNameArray, pos); + char** p2 = taosArrayGet(pNameArray, i); + + if (strcmp(*p1, *p2) == 0) { + // do nothing + } else { + if (pos + 1 != i) { + char* p = taosArrayGetP(pNameArray, pos + 1); + tfree(p); + taosArraySet(pNameArray, pos + 1, p2); + pos += 1; + } else { + pos += 1; + } + } + } + + for(int32_t i = pos + 1; i < pNameArray->size; ++i) { + char* p = taosArrayGetP(pNameArray, i); + tfree(p); + } + + pNameArray->size = pos + 1; return TSDB_CODE_SUCCESS; } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 4f04a82f0a..22f721d9e0 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -24,6 +24,35 @@ extern "C" { #endif +#pragma pack(push, 1) +typedef struct { + VarDataLenT len; + uint8_t data; +} SBinaryNullT; + +typedef struct { + VarDataLenT len; + uint32_t data; +} SNCharNullT; +#pragma pack(pop) + +extern const uint8_t BoolNull; +extern const uint8_t TinyintNull; +extern const uint16_t SmallintNull; +extern const uint32_t IntNull; +extern const uint64_t BigintNull; +extern const uint64_t TimestampNull; +extern const uint8_t UTinyintNull; +extern const uint16_t USmallintNull; +extern const uint32_t UIntNull; +extern const uint64_t UBigintNull; +extern const uint32_t FloatNull; +extern const uint64_t DoubleNull; +extern const SBinaryNullT BinaryNull; +extern const SNCharNullT NcharNull; + +const void *tdGetNullVal(int8_t type); + #define STR_TO_VARSTR(x, str) \ do { \ VarDataLenT __len = (VarDataLenT)strlen(str); \ @@ -53,10 +82,10 @@ extern "C" { // ----------------- TSDB COLUMN DEFINITION typedef struct { - int8_t type; // Column type - int16_t colId; // column ID - int16_t bytes; // column bytes - int16_t offset; // point offset in SDataRow after the header part + int8_t type; // Column type + int16_t colId; // column ID + uint16_t bytes; // column bytes + uint16_t offset; // point offset in SDataRow after the header part. } STColumn; #define colType(col) ((col)->type) @@ -167,10 +196,11 @@ static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) { return 0; } } + // ----------------- Data row structure /* A data row, the format is like below: - * |<--------------------+--------------------------- len ---------------------------------->| + * |<------------------------------------------------ len ---------------------------------->| * |<-- Head -->|<--------- flen -------------->| | * +---------------------+---------------------------------+---------------------------------+ * | uint16_t | int16_t | | | @@ -184,8 +214,8 @@ typedef void *SDataRow; #define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) -#define dataRowLen(r) (*(uint16_t *)(r)) -#define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)) +#define dataRowLen(r) (*(TDRowLenT *)(r)) // 0~65535 +#define dataRowVersion(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))) #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) #define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r))) #define dataRowKey(r) tdGetKey(dataRowTKey(r)) @@ -201,20 +231,19 @@ void tdInitDataRow(SDataRow row, STSchema *pSchema); SDataRow tdDataRowDup(SDataRow row); // offset here not include dataRow header length -static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset) { +static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) { ASSERT(value != NULL); int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE; - char * ptr = (char *)POINTER_SHIFT(row, dataRowLen(row)); if (IS_VAR_DATA_TYPE(type)) { *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row); - memcpy(ptr, value, varDataTLen(value)); + memcpy(POINTER_SHIFT(row, dataRowLen(row)), value, varDataTLen(value)); dataRowLen(row) += varDataTLen(value); } else { if (offset == 0) { ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP); TKEY tvalue = tdGetTKEY(*(TSKEY *)value); - memcpy(POINTER_SHIFT(row, toffset), (void *)(&tvalue), TYPE_BYTES[type]); + memcpy(POINTER_SHIFT(row, toffset), (const void *)(&tvalue), TYPE_BYTES[type]); } else { memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]); } @@ -245,17 +274,21 @@ typedef struct SDataCol { TSKEY ts; // only used in last NULL column } SDataCol; +#define isAllRowsNull(pCol) ((pCol)->len == 0) static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints); -void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints); +void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); void dataColSetOffset(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); // Get the data pointer from a column-wised data -static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) { +static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) { + if (isAllRowsNull(pCol)) { + return tdGetNullVal(pCol->type); + } if (IS_VAR_DATA_TYPE(pCol->type)) { return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); } else { @@ -287,7 +320,7 @@ typedef struct { } SDataCols; #define keyCol(pCols) (&((pCols)->cols[0])) // Key column -#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] +#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] // the idx row of column-wised data #define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx)) static FORCE_INLINE TKEY dataColsTKeyFirst(SDataCols *pCols) { if (pCols->numOfRows) { @@ -331,13 +364,13 @@ void tdResetDataCols(SDataCols *pCols); int tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); SDataCols *tdFreeDataCols(SDataCols *pCols); -void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols); int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset); // ----------------- K-V data row structure -/* +/* |<-------------------------------------- len -------------------------------------------->| + * |<----- header ----->|<--------------------------- body -------------------------------->| * +----------+----------+---------------------------------+---------------------------------+ - * | int16_t | int16_t | | | + * | uint16_t | int16_t | | | * +----------+----------+---------------------------------+---------------------------------+ * | len | ncols | cols index | data part | * +----------+----------+---------------------------------+---------------------------------+ @@ -345,14 +378,14 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge typedef void *SKVRow; typedef struct { - int16_t colId; - int16_t offset; + int16_t colId; + uint16_t offset; } SColIdx; -#define TD_KV_ROW_HEAD_SIZE (2 * sizeof(int16_t)) +#define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) -#define kvRowLen(r) (*(int16_t *)(r)) -#define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))) +#define kvRowLen(r) (*(TDRowLenT *)(r)) +#define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t))) #define kvRowSetLen(r, len) kvRowLen(r) = (len) #define kvRowSetNCols(r, n) kvRowNCols(r) = (n) #define kvRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE) @@ -362,6 +395,9 @@ typedef struct { #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) #define kvRowFree(r) tfree(r) #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) +#define kvRowTKey(r) (*(TKEY *)(kvRowValues(r))) +#define kvRowKey(r) tdGetKey(kvRowTKey(r)) +#define kvRowDeleted(r) TKEY_IS_DELETED(kvRowTKey(r)) SKVRow tdKVRowDup(SKVRow row); int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value); @@ -385,13 +421,44 @@ static FORCE_INLINE void *tdGetKVRowValOfCol(SKVRow row, int16_t colId) { return kvRowColVal(row, (SColIdx *)ret); } +static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) { + return taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ); +} + +// offset here not include kvRow header length +static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t colId, int8_t type, int32_t offset) { + ASSERT(value != NULL); + int32_t toffset = offset + TD_KV_ROW_HEAD_SIZE; + SColIdx *pColIdx = (SColIdx *)POINTER_SHIFT(row, toffset); + char * ptr = (char *)POINTER_SHIFT(row, kvRowLen(row)); + + pColIdx->colId = colId; + pColIdx->offset = kvRowLen(row); // offset of pColIdx including the TD_KV_ROW_HEAD_SIZE + + if (IS_VAR_DATA_TYPE(type)) { + memcpy(ptr, value, varDataTLen(value)); + kvRowLen(row) += varDataTLen(value); + } else { + if (offset == 0) { + ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP); + TKEY tvalue = tdGetTKEY(*(TSKEY *)value); + memcpy(ptr, (void *)(&tvalue), TYPE_BYTES[type]); + } else { + memcpy(ptr, value, TYPE_BYTES[type]); + } + kvRowLen(row) += TYPE_BYTES[type]; + } + + return 0; +} + // ----------------- K-V data row builder typedef struct { int16_t tCols; int16_t nCols; SColIdx *pColIdx; - int16_t alloc; - int16_t size; + uint16_t alloc; + uint16_t size; void * buf; } SKVRowBuilder; @@ -427,8 +494,146 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, return 0; } +// ----------------- SMemRow appended with sequential data row structure +/* + * |---------|------------------------------------------------- len ---------------------------------->| + * |<-------- Head ------>|<--------- flen -------------->| | + * |---------+---------------------+---------------------------------+---------------------------------+ + * | uint8_t | uint16_t | int16_t | | | + * |---------+----------+----------+---------------------------------+---------------------------------+ + * | flag | len | sversion | First part | Second part | + * +---------+----------+----------+---------------------------------+---------------------------------+ + * + * NOTE: timestamp in this row structure is TKEY instead of TSKEY + */ + +// ----------------- SMemRow appended with extended K-V data row structure +/* |--------------------|------------------------------------------------ len ---------------------------------->| + * |<------------- Head ------------>|<--------- flen -------------->| | + * |--------------------+----------+--------------------------------------------+---------------------------------+ + * | uint8_t | int16_t | uint16_t | int16_t | | | + * |---------+----------+----------+----------+---------------------------------+---------------------------------+ + * | flag | sversion | len | ncols | cols index | data part | + * |---------+----------+----------+----------+---------------------------------+---------------------------------+ + */ + +typedef void *SMemRow; + +#define TD_MEM_ROW_TYPE_SIZE sizeof(uint8_t) +#define TD_MEM_ROW_KV_VER_SIZE sizeof(int16_t) +#define TD_MEM_ROW_KV_TYPE_VER_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE) +#define TD_MEM_ROW_DATA_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE) +// #define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE) + +#define SMEM_ROW_DATA 0U // SDataRow +#define SMEM_ROW_KV 1U // SKVRow + +#define memRowType(r) (*(uint8_t *)(r)) +#define isDataRow(r) (SMEM_ROW_DATA == memRowType(r)) +#define isKvRow(r) (SMEM_ROW_KV == memRowType(r)) + +#define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE) // section after flag +#define memRowKvBody(r) \ + POINTER_SHIFT(r, TD_MEM_ROW_KV_TYPE_VER_SIZE) // section after flag + sversion as to reuse SKVRow + +#define memRowDataLen(r) (*(TDRowLenT *)memRowDataBody(r)) // 0~65535 +#define memRowKvLen(r) (*(TDRowLenT *)memRowKvBody(r)) // 0~65535 + +#define memRowDataTLen(r) \ + ((TDRowTLenT)(memRowDataLen(r) + TD_MEM_ROW_TYPE_SIZE)) // using uint32_t/int32_t to store the TLen + +#define memRowKvTLen(r) ((TDRowTLenT)(memRowKvLen(r) + TD_MEM_ROW_KV_TYPE_VER_SIZE)) + +#define memRowLen(r) (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r)) +#define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r)) // using uint32_t/int32_t to store the TLen + +#define memRowDataVersion(r) dataRowVersion(memRowDataBody(r)) +#define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) +#define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r)) // schema version +#define memRowSetKvVersion(r, v) (memRowKvVersion(r) = (v)) +#define memRowTuple(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowValues(memRowKvBody(r))) + +#define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowDataBody(r)) : kvRowTKey(memRowKvBody(r))) +#define memRowKey(r) (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r))) +#define memRowSetTKey(r, k) \ + do { \ + if (isDataRow(r)) { \ + dataRowTKey(memRowDataBody(r)) = (k); \ + } else { \ + kvRowTKey(memRowKvBody(r)) = (k); \ + } \ + } while (0) + +#define memRowSetType(r, t) (memRowType(r) = (t)) +#define memRowSetLen(r, l) (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l)) +#define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowKvSetVersion(r, v)) +#define memRowCpy(dst, r) memcpy((dst), (r), memRowTLen(r)) +#define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_DATA_HEAD_SIZE) +#define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r)) + +SMemRow tdMemRowDup(SMemRow row); +void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols); +// NOTE: offset here including the header size +static FORCE_INLINE void *tdGetKvRowDataOfCol(void *row, int32_t offset) { return POINTER_SHIFT(row, offset); } +// NOTE: offset here including the header size +static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int8_t type, int32_t offset) { + if (isDataRow(row)) { + return tdGetRowDataOfCol(row, type, offset); + } else if (isKvRow(row)) { + return tdGetKvRowDataOfCol(row, offset); + } else { + ASSERT(0); + } + return NULL; +} + +// ----------------- Raw payload structure for row: +/* |<------------ Head ------------->|<----------- body of column data tuple ------------------->| + * | |<----------------- flen ------------->|<--- value part --->| + * |SMemRowType| dataTLen | nCols | colId | colType | offset | ... | value |...|...|... | + * +-----------+----------+----------+--------------------------------------|--------------------| + * | uint8_t | uint32_t | uint16_t | int16_t | uint8_t | uint16_t | ... |.......|...|...|... | + * +-----------+----------+----------+--------------------------------------+--------------------| + * 1. offset in column data tuple starts from the value part in case of uint16_t overflow. + * 2. dataTLen: total length including the header and body. + */ + +#define PAYLOAD_NCOLS_LEN sizeof(uint16_t) +#define PAYLOAD_NCOLS_OFFSET (sizeof(uint8_t) + sizeof(TDRowTLenT)) +#define PAYLOAD_HEADER_LEN (PAYLOAD_NCOLS_OFFSET + PAYLOAD_NCOLS_LEN) +#define PAYLOAD_ID_LEN sizeof(int16_t) +#define PAYLOAD_ID_TYPE_LEN (sizeof(int16_t) + sizeof(uint8_t)) +#define PAYLOAD_COL_HEAD_LEN (PAYLOAD_ID_TYPE_LEN + sizeof(uint16_t)) +#define PAYLOAD_PRIMARY_COL_LEN (PAYLOAD_ID_TYPE_LEN + sizeof(TSKEY)) + +#define payloadBody(r) POINTER_SHIFT(r, PAYLOAD_HEADER_LEN) +#define payloadType(r) (*(uint8_t *)(r)) +#define payloadSetType(r, t) (payloadType(r) = (t)) +#define payloadTLen(r) (*(TDRowTLenT *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) // including total header +#define payloadSetTLen(r, l) (payloadTLen(r) = (l)) +#define payloadNCols(r) (*(TDRowLenT *)POINTER_SHIFT(r, PAYLOAD_NCOLS_OFFSET)) +#define payloadSetNCols(r, n) (payloadNCols(r) = (n)) +#define payloadValuesOffset(r) \ + (PAYLOAD_HEADER_LEN + payloadNCols(r) * PAYLOAD_COL_HEAD_LEN) // avoid using the macro in loop +#define payloadValues(r) POINTER_SHIFT(r, payloadValuesOffset(r)) // avoid using the macro in loop +#define payloadColId(c) (*(int16_t *)(c)) +#define payloadColType(c) (*(uint8_t *)POINTER_SHIFT(c, PAYLOAD_ID_LEN)) +#define payloadColOffset(c) (*(uint16_t *)POINTER_SHIFT(c, PAYLOAD_ID_TYPE_LEN)) +#define payloadColValue(c) POINTER_SHIFT(c, payloadColOffset(c)) + +#define payloadColSetId(c, i) (payloadColId(c) = (i)) +#define payloadColSetType(c, t) (payloadColType(c) = (t)) +#define payloadColSetOffset(c, o) (payloadColOffset(c) = (o)) + +#define payloadTSKey(r) (*(TSKEY *)POINTER_SHIFT(r, payloadValuesOffset(r))) +#define payloadTKey(r) (*(TKEY *)POINTER_SHIFT(r, payloadValuesOffset(r))) +#define payloadKey(r) tdGetKey(payloadTKey(r)) + + +static FORCE_INLINE char *payloadNextCol(char *pCol) { return (char *)POINTER_SHIFT(pCol, PAYLOAD_COL_HEAD_LEN); } + #ifdef __cplusplus } #endif -#endif // _TD_DATA_FORMAT_H_ +#endif // _TD_DATA_FORMAT_H_ \ No newline at end of file diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 94c429cfc0..04d70e7aa0 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -18,6 +18,21 @@ #include "tcoding.h" #include "wchar.h" +const uint8_t BoolNull = TSDB_DATA_BOOL_NULL; +const uint8_t TinyintNull = TSDB_DATA_TINYINT_NULL; +const uint16_t SmallintNull = TSDB_DATA_SMALLINT_NULL; +const uint32_t IntNull = TSDB_DATA_INT_NULL; +const uint64_t BigintNull = TSDB_DATA_BIGINT_NULL; +const uint64_t TimestampNull = TSDB_DATA_BIGINT_NULL; +const uint8_t UTinyintNull = TSDB_DATA_UTINYINT_NULL; +const uint16_t USmallintNull = TSDB_DATA_USMALLINT_NULL; +const uint32_t UIntNull = TSDB_DATA_UINT_NULL; +const uint64_t UBigintNull = TSDB_DATA_UBIGINT_NULL; +const uint32_t FloatNull = TSDB_DATA_FLOAT_NULL; +const uint64_t DoubleNull = TSDB_DATA_DOUBLE_NULL; +const SBinaryNullT BinaryNull = {1, TSDB_DATA_BINARY_NULL}; +const SNCharNullT NcharNull = {4, TSDB_DATA_NCHAR_NULL}; + static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows); @@ -198,6 +213,14 @@ SDataRow tdDataRowDup(SDataRow row) { return trow; } +SMemRow tdMemRowDup(SMemRow row) { + SMemRow trow = malloc(memRowTLen(row)); + if (trow == NULL) return NULL; + + memRowCpy(trow, row); + return trow; +} + void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) { pDataCol->type = colType(pCol); pDataCol->colId = colColId(pCol); @@ -217,11 +240,22 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) *pBuf = POINTER_SHIFT(*pBuf, pDataCol->spaceSize); } } - // value from timestamp should be TKEY here instead of TSKEY -void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) { +void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { ASSERT(pCol != NULL && value != NULL); + if (isAllRowsNull(pCol)) { + if (isNull(value, pCol->type)) { + // all null value yet, just return + return; + } + + if (numOfRows > 0) { + // Find the first not null value, fill all previouse values as NULL + dataColSetNEleNull(pCol, numOfRows, maxPoints); + } + } + if (IS_VAR_DATA_TYPE(pCol->type)) { // set offset pCol->dataOff[numOfRows] = pCol->len; @@ -243,7 +277,7 @@ bool isNEleNull(SDataCol *pCol, int nEle) { return true; } -void dataColSetNullAt(SDataCol *pCol, int index) { +FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->dataOff[index] = pCol->len; char *ptr = POINTER_SHIFT(pCol->pData, pCol->len); @@ -399,8 +433,7 @@ void tdResetDataCols(SDataCols *pCols) { } } } - -void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) { +static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) { ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row)); int rcol = 0; @@ -419,7 +452,8 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) while (dcol < pCols->numOfCols) { SDataCol *pDataCol = &(pCols->cols[dcol]); if (rcol >= schemaNCols(pSchema)) { - dataColSetNullAt(pDataCol, pCols->numOfRows); + // dataColSetNullAt(pDataCol, pCols->numOfRows); + dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); dcol++; continue; } @@ -433,7 +467,8 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) } else if (pRowCol->colId < pDataCol->colId) { rcol++; } else { - dataColSetNullAt(pDataCol, pCols->numOfRows); + // dataColSetNullAt(pDataCol, pCols->numOfRows); + dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); dcol++; } } @@ -441,6 +476,62 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) pCols->numOfRows++; } +static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCols) { + ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < kvRowKey(row)); + + int rcol = 0; + int dcol = 0; + + if (kvRowDeleted(row)) { + for (; dcol < pCols->numOfCols; dcol++) { + SDataCol *pDataCol = &(pCols->cols[dcol]); + if (dcol == 0) { + dataColAppendVal(pDataCol, kvRowValues(row), pCols->numOfRows, pCols->maxPoints); + } else { + dataColSetNullAt(pDataCol, pCols->numOfRows); + } + } + } else { + int nRowCols = kvRowNCols(row); + + while (dcol < pCols->numOfCols) { + SDataCol *pDataCol = &(pCols->cols[dcol]); + if (rcol >= nRowCols || rcol >= schemaNCols(pSchema)) { + // dataColSetNullAt(pDataCol, pCols->numOfRows); + dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + ++dcol; + continue; + } + + SColIdx *colIdx = kvRowColIdxAt(row, rcol); + + if (colIdx->colId == pDataCol->colId) { + void *value = tdGetKvRowDataOfCol(row, colIdx->offset); + dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints); + ++dcol; + ++rcol; + } else if (colIdx->colId < pDataCol->colId) { + ++rcol; + } else { + // dataColSetNullAt(pDataCol, pCols->numOfRows); + dataColAppendVal(pDataCol, tdGetNullVal(pDataCol->type), pCols->numOfRows, pCols->maxPoints); + ++dcol; + } + } + } + pCols->numOfRows++; +} + +void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols) { + if (isDataRow(row)) { + tdAppendDataRowToDataCol(memRowDataBody(row), pSchema, pCols); + } else if (isKvRow(row)) { + tdAppendKvRowToDataCol(memRowKvBody(row), pSchema, pCols); + } else { + ASSERT(0); + } +} + int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset) { ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows); ASSERT(target->numOfCols == source->numOfCols); @@ -563,7 +654,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { nrow = malloc(kvRowLen(row) + sizeof(SColIdx) + diff); if (nrow == NULL) return -1; - kvRowSetLen(nrow, kvRowLen(row) + (int16_t)sizeof(SColIdx) + diff); + kvRowSetLen(nrow, kvRowLen(row) + (uint16_t)sizeof(SColIdx) + diff); kvRowSetNCols(nrow, kvRowNCols(row) + 1); if (ptr == NULL) { @@ -605,8 +696,8 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { if (varDataTLen(value) == varDataTLen(pOldVal)) { // just update the column value in place memcpy(pOldVal, value, varDataTLen(value)); } else { // need to reallocate the memory - int16_t diff = varDataTLen(value) - varDataTLen(pOldVal); - int16_t nlen = kvRowLen(row) + diff; + uint16_t diff = varDataTLen(value) - varDataTLen(pOldVal); + uint16_t nlen = kvRowLen(row) + diff; ASSERT(nlen > 0); nrow = malloc(nlen); if (nrow == NULL) return -1; @@ -709,3 +800,39 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { return row; } + +const void *tdGetNullVal(int8_t type) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + return &BoolNull; + case TSDB_DATA_TYPE_TINYINT: + return &TinyintNull; + case TSDB_DATA_TYPE_SMALLINT: + return &SmallintNull; + case TSDB_DATA_TYPE_INT: + return &IntNull; + case TSDB_DATA_TYPE_BIGINT: + return &BigintNull; + case TSDB_DATA_TYPE_FLOAT: + return &FloatNull; + case TSDB_DATA_TYPE_DOUBLE: + return &DoubleNull; + case TSDB_DATA_TYPE_BINARY: + return &BinaryNull; + case TSDB_DATA_TYPE_TIMESTAMP: + return &TimestampNull; + case TSDB_DATA_TYPE_NCHAR: + return &NcharNull; + case TSDB_DATA_TYPE_UTINYINT: + return &UTinyintNull; + case TSDB_DATA_TYPE_USMALLINT: + return &USmallintNull; + case TSDB_DATA_TYPE_UINT: + return &UIntNull; + case TSDB_DATA_TYPE_UBIGINT: + return &UBigintNull; + default: + ASSERT(0); + return NULL; + } +} \ No newline at end of file diff --git a/src/connector/go b/src/connector/go index 7a26c432f8..b8f76da4a7 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 7a26c432f8b4203e42344ff3290b9b9b01b983d5 +Subproject commit b8f76da4a708d158ec3cc4b844571dc4414e36b4 diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index 23cf0b9335..3530c6df09 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit 23cf0b933510af5e87b61bace0f6ff2f06d8eaac +Subproject commit 3530c6df097134a410bacec6b3cd013ef38a61aa diff --git a/src/connector/hivemq-tdengine-extension b/src/connector/hivemq-tdengine-extension index b62a26ecc1..ce52010141 160000 --- a/src/connector/hivemq-tdengine-extension +++ b/src/connector/hivemq-tdengine-extension @@ -1 +1 @@ -Subproject commit b62a26ecc164a310104df57691691b237e091c89 +Subproject commit ce5201014136503d34fecbd56494b67b4961056c diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java index b91fe88dfa..2ff0d86c92 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java @@ -19,6 +19,7 @@ import com.taosdata.jdbc.utils.NullType; import java.math.BigDecimal; import java.sql.SQLException; import java.sql.Timestamp; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; @@ -463,6 +464,25 @@ public class TSDBResultSetRowData { data.set(col, tsObj); } + /** + * this implementation is used for TDengine old version + */ + public void setTimestamp(int col, long ts) { + //TODO: this implementation contains logical error + // when precision is us the (long ts) is 16 digital number + // when precision is ms, the (long ts) is 13 digital number + // we need a JNI function like this: + // public void setTimestamp(int col, long epochSecond, long nanoAdjustment) + if (ts < 1_0000_0000_0000_0L) { + data.set(col, new Timestamp(ts)); + } else { + long epochSec = ts / 1000_000L; + long nanoAdjustment = ts % 1000_000L * 1000L; + Timestamp timestamp = Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment)); + data.set(col, timestamp); + } + } + public Timestamp getTimestamp(int col, int nativeType) { Object obj = data.get(col - 1); if (obj == null) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java index efe3303bd9..c91bacbfdc 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java @@ -5,7 +5,6 @@ import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; import com.taosdata.jdbc.enums.TimestampPrecision; -import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.sql.Date; import java.sql.Time; @@ -110,15 +109,27 @@ public class Utils { return rawSql; // toLowerCase String preparedSql = rawSql.trim().toLowerCase(); - String[] clause = new String[]{"values\\s*\\([\\s\\S]*?\\)", "tags\\s*\\([\\s\\S]*?\\)", "where[\\s\\S]*"}; + String[] clause = new String[]{"tags\\s*\\([\\s\\S]*?\\)", "where[\\s\\S]*"}; Map placeholderPositions = new HashMap<>(); RangeSet clauseRangeSet = TreeRangeSet.create(); findPlaceholderPosition(preparedSql, placeholderPositions); + // find tags and where clause's position findClauseRangeSet(preparedSql, clause, clauseRangeSet); + // find values clause's position + findValuesClauseRangeSet(preparedSql, clauseRangeSet); return transformSql(rawSql, parameters, placeholderPositions, clauseRangeSet); } + private static void findValuesClauseRangeSet(String preparedSql, RangeSet clauseRangeSet) { + Matcher matcher = Pattern.compile("(values|,)\\s*(\\([^)]*\\))").matcher(preparedSql); + while (matcher.find()) { + int start = matcher.start(2); + int end = matcher.end(2); + clauseRangeSet.add(Range.closedOpen(start, end)); + } + } + private static void findClauseRangeSet(String preparedSql, String[] regexArr, RangeSet clauseRangeSet) { clauseRangeSet.clear(); for (String regex : regexArr) { @@ -126,7 +137,7 @@ public class Utils { while (matcher.find()) { int start = matcher.start(); int end = matcher.end(); - clauseRangeSet.add(Range.closed(start, end)); + clauseRangeSet.add(Range.closedOpen(start, end)); } } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java index ee83f6ceb2..4b8ec61078 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java @@ -3,6 +3,8 @@ package com.taosdata.jdbc.utils; import org.junit.Assert; import org.junit.Test; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Stream; public class UtilsTest { @@ -32,7 +34,7 @@ public class UtilsTest { } @Test - public void getNativeSqlReplaceQuestionMarks() { + public void lowerCase() { // given String nativeSql = "insert into ?.? (ts, temperature, humidity) using ?.? tags(?,?) values(now, ?, ?)"; Object[] parameters = Stream.of("test", "t1", "test", "weather", "beijing", 1, 12.2, 4).toArray(); @@ -46,7 +48,7 @@ public class UtilsTest { } @Test - public void getNativeSqlReplaceQuestionMarks2() { + public void upperCase() { // given String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?) VALUES (?,?,?,?)"; Object[] parameters = Stream.of("d1", 1, 123, 3.14, 220, 4).toArray(); @@ -59,9 +61,49 @@ public class UtilsTest { Assert.assertEquals(expected, actual); } + @Test + public void multiValues() { + // given + String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?) VALUES (?,?,?,?),(?,?,?,?)"; + Object[] parameters = Stream.of("d1", 1, 100, 3.14, "abc", 4, 200, 3.1415, "xyz", 5).toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1) VALUES (100,3.14,'abc',4),(200,3.1415,'xyz',5)"; + Assert.assertEquals(expected, actual); + } @Test - public void getNativeSqlReplaceNothing() { + public void lineTerminator() { + // given + String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?) VALUES (?,?,\r\n?,?),(?,?,?,?)"; + Object[] parameters = Stream.of("d1", 1, 100, 3.14, "abc", 4, 200, 3.1415, "xyz", 5).toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1) VALUES (100,3.14,\r\n'abc',4),(200,3.1415,'xyz',5)"; + Assert.assertEquals(expected, actual); + } + + @Test + public void lineTerminatorAndMultiValues() { + String nativeSql = "INSERT Into ? TAGS(?) VALUES(?,?,\r\n?,?),(?,? ,\r\n?,?) t? tags (?) Values (?,?,?\r\n,?),(?,?,?,?) t? Tags(?) values (?,?,?,?) , (?,?,?,?)"; + Object[] parameters = Stream.of("t1", "abc", 100, 1.1, "xxx", "xxx", 200, 2.2, "xxx", "xxx", 2, "bcd", 300, 3.3, "xxx", "xxx", 400, 4.4, "xxx", "xxx", 3, "cde", 500, 5.5, "xxx", "xxx", 600, 6.6, "xxx", "xxx").toArray(); + + // when + String actual = Utils.getNativeSql(nativeSql, parameters); + + // then + String expected = "INSERT Into t1 TAGS('abc') VALUES(100,1.1,\r\n'xxx','xxx'),(200,2.2 ,\r\n'xxx','xxx') t2 tags ('bcd') Values (300,3.3,'xxx'\r\n,'xxx'),(400,4.4,'xxx','xxx') t3 Tags('cde') values (500,5.5,'xxx','xxx') , (600,6.6,'xxx','xxx')"; + Assert.assertEquals(expected, actual); + } + + @Test + public void replaceNothing() { // given String nativeSql = "insert into test.t1 (ts, temperature, humidity) using test.weather tags('beijing',1) values(now, 12.2, 4)"; @@ -73,7 +115,7 @@ public class UtilsTest { } @Test - public void getNativeSqlReplaceNothing2() { + public void replaceNothing2() { // given String nativeSql = "insert into test.t1 (ts, temperature, humidity) using test.weather tags('beijing',1) values(now, 12.2, 4)"; Object[] parameters = Stream.of("test", "t1", "test", "weather", "beijing", 1, 12.2, 4).toArray(); @@ -86,7 +128,7 @@ public class UtilsTest { } @Test - public void getNativeSqlReplaceNothing3() { + public void replaceNothing3() { // given String nativeSql = "insert into ?.? (ts, temperature, humidity) using ?.? tags(?,?) values(now, ?, ?)"; diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index f539e77253..cd762f3110 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -476,15 +476,17 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { cDebug("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr); - int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize; + int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_MEM_ROW_DATA_HEAD_SIZE + pObj->rowSize; char *buffer = calloc(size, 1); SWalHead *pHead = (SWalHead *)buffer; SSubmitMsg *pMsg = (SSubmitMsg *) (buffer + sizeof(SWalHead)); SSubmitBlk *pBlk = (SSubmitBlk *) (buffer + sizeof(SWalHead) + sizeof(SSubmitMsg)); - SDataRow trow = (SDataRow)pBlk->data; - tdInitDataRow(trow, pSchema); + SMemRow trow = (SMemRow)pBlk->data; + SDataRow dataRow = (SDataRow)memRowDataBody(trow); + memRowSetType(trow, SMEM_ROW_DATA); + tdInitDataRow(dataRow, pSchema); for (int32_t i = 0; i < pSchema->numOfCols; i++) { STColumn *c = pSchema->columns + i; @@ -500,9 +502,9 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { memcpy((char *)val + sizeof(VarDataLenT), buf, len); varDataLen(val) = len; } - tdAppendColVal(trow, val, c->type, c->bytes, c->offset); + tdAppendColVal(dataRow, val, c->type, c->offset); } - pBlk->dataLen = htonl(dataRowLen(trow)); + pBlk->dataLen = htonl(memRowDataTLen(trow)); pBlk->schemaLen = 0; pBlk->uid = htobe64(pObj->uid); @@ -511,7 +513,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { pBlk->sversion = htonl(pSchema->version); pBlk->padding = 0; - pHead->len = sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + dataRowLen(trow); + pHead->len = sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + memRowDataTLen(trow); pMsg->header.vgId = htonl(pContext->vgId); pMsg->header.contLen = htonl(pHead->len); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 56118b0a76..ca8ad3cc09 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -180,7 +180,7 @@ do { \ // this is the length of its string representation, including the terminator zero #define TSDB_ACCT_ID_LEN 11 -#define TSDB_MAX_COLUMNS 1024 +#define TSDB_MAX_COLUMNS 4096 #define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns #define TSDB_NODE_NAME_LEN 64 @@ -199,7 +199,13 @@ do { \ #define TSDB_APPNAME_LEN TSDB_UNI_LEN -#define TSDB_MAX_BYTES_PER_ROW 16384 + /** + * In some scenarios uint16_t (0~65535) is used to store the row len. + * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. + * - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus + * the final value is 65531-(4096-1)*4 = 49151. + */ +#define TSDB_MAX_BYTES_PER_ROW 49151 #define TSDB_MAX_TAGS_LEN 16384 #define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAG_CONDITIONS 1024 @@ -327,8 +333,9 @@ do { \ #define TSDB_MAX_JOIN_TABLE_NUM 10 #define TSDB_MAX_UNION_CLAUSE 5 -#define TSDB_MAX_BINARY_LEN (TSDB_MAX_BYTES_PER_ROW-TSDB_KEYSIZE) -#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_BYTES_PER_ROW-TSDB_KEYSIZE) +#define TSDB_MAX_FIELD_LEN 16384 +#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 +#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 #define PRIMARYKEY_TIMESTAMP_COL_INDEX 0 #define TSDB_MAX_RPC_THREADS 5 diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 0645f2e8b3..f57e553e3f 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -175,6 +175,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_FIELD_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x036C) //"Field does not exist") #define TSDB_CODE_MND_INVALID_STABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x036D) //"Super table does not exist") #define TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG TAOS_DEF_ERROR_CODE(0, 0x036E) //"Invalid create table message") +#define TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x036F) //"Exceed max row bytes") #define TSDB_CODE_MND_INVALID_FUNC_NAME TAOS_DEF_ERROR_CODE(0, 0x0370) //"Invalid func name") #define TSDB_CODE_MND_INVALID_FUNC_LEN TAOS_DEF_ERROR_CODE(0, 0x0371) //"Invalid func length") diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 1ad4e18bd0..9ee241efc1 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -751,6 +751,7 @@ typedef struct { } STableInfoMsg; typedef struct { + uint8_t metaClone; // create local clone of the cached table meta int32_t numOfVgroups; int32_t numOfTables; int32_t numOfUdfs; @@ -808,6 +809,7 @@ typedef struct SMultiTableMeta { int32_t contLen; uint8_t compressed; // denote if compressed or not uint32_t rawLen; // size before compress + uint8_t metaClone; // make meta clone after retrieve meta from mnode char meta[]; } SMultiTableMeta; diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 8eaa52f90a..8def1cefe2 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -10,8 +10,10 @@ extern "C" { #include "taosdef.h" // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR -typedef int32_t VarDataOffsetT; -typedef int16_t VarDataLenT; +typedef int32_t VarDataOffsetT; +typedef int16_t VarDataLenT; // maxVarDataLen: 32767 +typedef uint16_t TDRowLenT; // not including overhead: 0 ~ 65535 +typedef uint32_t TDRowTLenT; // total length, including overhead typedef struct tstr { VarDataLenT len; diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 9e8526bfe4..964f1a1d0f 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -54,6 +54,7 @@ #include "tutil.h" #define STMT_IFACE_ENABLED 1 +#define NANO_SECOND_ENABLED 1 #define REQ_EXTRA_BUF_LEN 1024 #define RESP_BUF_LEN 4096 @@ -66,13 +67,6 @@ extern char configDir[]; #define STR_INSERT_INTO "INSERT INTO " -enum TEST_MODE { - INSERT_TEST, // 0 - QUERY_TEST, // 1 - SUBSCRIBE_TEST, // 2 - INVAID_TEST -}; - #define MAX_RECORDS_PER_REQ 32766 #define HEAD_BUFF_LEN TSDB_MAX_COLUMNS*24 // 16*MAX_COLUMNS + (192+32)*2 + insert into .. @@ -105,6 +99,13 @@ enum TEST_MODE { #define DEFAULT_TIMESTAMP_STEP 1 +enum TEST_MODE { + INSERT_TEST, // 0 + QUERY_TEST, // 1 + SUBSCRIBE_TEST, // 2 + INVAID_TEST +}; + typedef enum CREATE_SUB_TALBE_MOD_EN { PRE_CREATE_SUBTBL, AUTO_CREATE_SUBTBL, @@ -112,15 +113,15 @@ typedef enum CREATE_SUB_TALBE_MOD_EN { } CREATE_SUB_TALBE_MOD_EN; typedef enum TALBE_EXISTS_EN { - TBL_NO_EXISTS, - TBL_ALREADY_EXISTS, - TBL_EXISTS_BUTT + TBL_NO_EXISTS, + TBL_ALREADY_EXISTS, + TBL_EXISTS_BUTT } TALBE_EXISTS_EN; enum enumSYNC_MODE { - SYNC_MODE, - ASYNC_MODE, - MODE_BUT + SYNC_MODE, + ASYNC_MODE, + MODE_BUT }; enum enum_TAOS_INTERFACE { @@ -143,52 +144,52 @@ typedef enum enum_PROGRESSIVE_OR_INTERLACE { } PROG_OR_INTERLACE_MODE; typedef enum enumQUERY_TYPE { - NO_INSERT_TYPE, - INSERT_TYPE, - QUERY_TYPE_BUT + NO_INSERT_TYPE, + INSERT_TYPE, + QUERY_TYPE_BUT } QUERY_TYPE; enum _show_db_index { - TSDB_SHOW_DB_NAME_INDEX, - TSDB_SHOW_DB_CREATED_TIME_INDEX, - TSDB_SHOW_DB_NTABLES_INDEX, - TSDB_SHOW_DB_VGROUPS_INDEX, - TSDB_SHOW_DB_REPLICA_INDEX, - TSDB_SHOW_DB_QUORUM_INDEX, - TSDB_SHOW_DB_DAYS_INDEX, - TSDB_SHOW_DB_KEEP_INDEX, - TSDB_SHOW_DB_CACHE_INDEX, - TSDB_SHOW_DB_BLOCKS_INDEX, - TSDB_SHOW_DB_MINROWS_INDEX, - TSDB_SHOW_DB_MAXROWS_INDEX, - TSDB_SHOW_DB_WALLEVEL_INDEX, - TSDB_SHOW_DB_FSYNC_INDEX, - TSDB_SHOW_DB_COMP_INDEX, - TSDB_SHOW_DB_CACHELAST_INDEX, - TSDB_SHOW_DB_PRECISION_INDEX, - TSDB_SHOW_DB_UPDATE_INDEX, - TSDB_SHOW_DB_STATUS_INDEX, - TSDB_MAX_SHOW_DB + TSDB_SHOW_DB_NAME_INDEX, + TSDB_SHOW_DB_CREATED_TIME_INDEX, + TSDB_SHOW_DB_NTABLES_INDEX, + TSDB_SHOW_DB_VGROUPS_INDEX, + TSDB_SHOW_DB_REPLICA_INDEX, + TSDB_SHOW_DB_QUORUM_INDEX, + TSDB_SHOW_DB_DAYS_INDEX, + TSDB_SHOW_DB_KEEP_INDEX, + TSDB_SHOW_DB_CACHE_INDEX, + TSDB_SHOW_DB_BLOCKS_INDEX, + TSDB_SHOW_DB_MINROWS_INDEX, + TSDB_SHOW_DB_MAXROWS_INDEX, + TSDB_SHOW_DB_WALLEVEL_INDEX, + TSDB_SHOW_DB_FSYNC_INDEX, + TSDB_SHOW_DB_COMP_INDEX, + TSDB_SHOW_DB_CACHELAST_INDEX, + TSDB_SHOW_DB_PRECISION_INDEX, + TSDB_SHOW_DB_UPDATE_INDEX, + TSDB_SHOW_DB_STATUS_INDEX, + TSDB_MAX_SHOW_DB }; // -----------------------------------------SHOW TABLES CONFIGURE ------------------------------------- enum _show_stables_index { - TSDB_SHOW_STABLES_NAME_INDEX, - TSDB_SHOW_STABLES_CREATED_TIME_INDEX, - TSDB_SHOW_STABLES_COLUMNS_INDEX, - TSDB_SHOW_STABLES_METRIC_INDEX, - TSDB_SHOW_STABLES_UID_INDEX, - TSDB_SHOW_STABLES_TID_INDEX, - TSDB_SHOW_STABLES_VGID_INDEX, - TSDB_MAX_SHOW_STABLES + TSDB_SHOW_STABLES_NAME_INDEX, + TSDB_SHOW_STABLES_CREATED_TIME_INDEX, + TSDB_SHOW_STABLES_COLUMNS_INDEX, + TSDB_SHOW_STABLES_METRIC_INDEX, + TSDB_SHOW_STABLES_UID_INDEX, + TSDB_SHOW_STABLES_TID_INDEX, + TSDB_SHOW_STABLES_VGID_INDEX, + TSDB_MAX_SHOW_STABLES }; enum _describe_table_index { - TSDB_DESCRIBE_METRIC_FIELD_INDEX, - TSDB_DESCRIBE_METRIC_TYPE_INDEX, - TSDB_DESCRIBE_METRIC_LENGTH_INDEX, - TSDB_DESCRIBE_METRIC_NOTE_INDEX, - TSDB_MAX_DESCRIBE_METRIC + TSDB_DESCRIBE_METRIC_FIELD_INDEX, + TSDB_DESCRIBE_METRIC_TYPE_INDEX, + TSDB_DESCRIBE_METRIC_LENGTH_INDEX, + TSDB_DESCRIBE_METRIC_NOTE_INDEX, + TSDB_MAX_DESCRIBE_METRIC }; /* Used by main to communicate with parse_opt. */ @@ -228,7 +229,7 @@ typedef struct SArguments_S { int64_t num_of_DPT; int abort; uint32_t disorderRatio; // 0: no disorder, >0: x% - int disorderRange; // ms or us by database precision + int disorderRange; // ms, us or ns. accordig to database precision uint32_t method_of_delete; char ** arg_list; uint64_t totalInsertRows; @@ -237,191 +238,191 @@ typedef struct SArguments_S { } SArguments; typedef struct SColumn_S { - char field[TSDB_COL_NAME_LEN]; - char dataType[16]; - uint32_t dataLen; - char note[128]; + char field[TSDB_COL_NAME_LEN]; + char dataType[16]; + uint32_t dataLen; + char note[128]; } StrColumn; typedef struct SSuperTable_S { - char sTblName[TSDB_TABLE_NAME_LEN]; - char dataSource[MAX_TB_NAME_SIZE]; // rand_gen or sample - char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq - char insertMode[MAX_TB_NAME_SIZE]; // taosc, rest - uint16_t childTblExists; - int64_t childTblCount; - uint64_t batchCreateTableNum; // 0: no batch, > 0: batch table number in one sql - uint8_t autoCreateTable; // 0: create sub table, 1: auto create sub table - uint16_t iface; // 0: taosc, 1: rest, 2: stmt - int64_t childTblLimit; - uint64_t childTblOffset; + char sTblName[TSDB_TABLE_NAME_LEN]; + char dataSource[MAX_TB_NAME_SIZE]; // rand_gen or sample + char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq + char insertMode[MAX_TB_NAME_SIZE]; // taosc, rest + uint16_t childTblExists; + int64_t childTblCount; + uint64_t batchCreateTableNum; // 0: no batch, > 0: batch table number in one sql + uint8_t autoCreateTable; // 0: create sub table, 1: auto create sub table + uint16_t iface; // 0: taosc, 1: rest, 2: stmt + int64_t childTblLimit; + uint64_t childTblOffset; -// int multiThreadWriteOneTbl; // 0: no, 1: yes - uint32_t interlaceRows; // - int disorderRatio; // 0: no disorder, >0: x% - int disorderRange; // ms or us by database precision - uint64_t maxSqlLen; // + // int multiThreadWriteOneTbl; // 0: no, 1: yes + uint32_t interlaceRows; // + int disorderRatio; // 0: no disorder, >0: x% + int disorderRange; // ms, us or ns. according to database precision + uint64_t maxSqlLen; // - uint64_t insertInterval; // insert interval, will override global insert interval - int64_t insertRows; - int64_t timeStampStep; - char startTimestamp[MAX_TB_NAME_SIZE]; - char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json - char sampleFile[MAX_FILE_NAME_LEN]; - char tagsFile[MAX_FILE_NAME_LEN]; + uint64_t insertInterval; // insert interval, will override global insert interval + int64_t insertRows; + int64_t timeStampStep; + char startTimestamp[MAX_TB_NAME_SIZE]; + char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json + char sampleFile[MAX_FILE_NAME_LEN]; + char tagsFile[MAX_FILE_NAME_LEN]; - uint32_t columnCount; - StrColumn columns[TSDB_MAX_COLUMNS]; - uint32_t tagCount; - StrColumn tags[TSDB_MAX_TAGS]; + uint32_t columnCount; + StrColumn columns[TSDB_MAX_COLUMNS]; + uint32_t tagCount; + StrColumn tags[TSDB_MAX_TAGS]; - char* childTblName; - char* colsOfCreateChildTable; - uint64_t lenOfOneRow; - uint64_t lenOfTagOfOneRow; + char* childTblName; + char* colsOfCreateChildTable; + uint64_t lenOfOneRow; + uint64_t lenOfTagOfOneRow; - char* sampleDataBuf; - //int sampleRowCount; - //int sampleUsePos; + char* sampleDataBuf; + //int sampleRowCount; + //int sampleUsePos; - uint32_t tagSource; // 0: rand, 1: tag sample - char* tagDataBuf; - uint32_t tagSampleCount; - uint32_t tagUsePos; + uint32_t tagSource; // 0: rand, 1: tag sample + char* tagDataBuf; + uint32_t tagSampleCount; + uint32_t tagUsePos; - // statistics - uint64_t totalInsertRows; - uint64_t totalAffectedRows; + // statistics + uint64_t totalInsertRows; + uint64_t totalAffectedRows; } SSuperTable; typedef struct { - char name[TSDB_DB_NAME_LEN]; - char create_time[32]; - int64_t ntables; - int32_t vgroups; - int16_t replica; - int16_t quorum; - int16_t days; - char keeplist[32]; - int32_t cache; //MB - int32_t blocks; - int32_t minrows; - int32_t maxrows; - int8_t wallevel; - int32_t fsync; - int8_t comp; - int8_t cachelast; - char precision[8]; // time resolution - int8_t update; - char status[16]; + char name[TSDB_DB_NAME_LEN]; + char create_time[32]; + int64_t ntables; + int32_t vgroups; + int16_t replica; + int16_t quorum; + int16_t days; + char keeplist[32]; + int32_t cache; //MB + int32_t blocks; + int32_t minrows; + int32_t maxrows; + int8_t wallevel; + int32_t fsync; + int8_t comp; + int8_t cachelast; + char precision[8]; // time resolution + int8_t update; + char status[16]; } SDbInfo; typedef struct SDbCfg_S { -// int maxtablesPerVnode; - uint32_t minRows; // 0 means default - uint32_t maxRows; // 0 means default - int comp; - int walLevel; - int cacheLast; - int fsync; - int replica; - int update; - int keep; - int days; - int cache; - int blocks; - int quorum; - char precision[8]; + // int maxtablesPerVnode; + uint32_t minRows; // 0 means default + uint32_t maxRows; // 0 means default + int comp; + int walLevel; + int cacheLast; + int fsync; + int replica; + int update; + int keep; + int days; + int cache; + int blocks; + int quorum; + char precision[8]; } SDbCfg; typedef struct SDataBase_S { - char dbName[TSDB_DB_NAME_LEN]; - bool drop; // 0: use exists, 1: if exists, drop then new create - SDbCfg dbCfg; - uint64_t superTblCount; - SSuperTable superTbls[MAX_SUPER_TABLE_COUNT]; + char dbName[TSDB_DB_NAME_LEN]; + bool drop; // 0: use exists, 1: if exists, drop then new create + SDbCfg dbCfg; + uint64_t superTblCount; + SSuperTable superTbls[MAX_SUPER_TABLE_COUNT]; } SDataBase; typedef struct SDbs_S { - char cfgDir[MAX_FILE_NAME_LEN]; - char host[MAX_HOSTNAME_SIZE]; - struct sockaddr_in serv_addr; + char cfgDir[MAX_FILE_NAME_LEN]; + char host[MAX_HOSTNAME_SIZE]; + struct sockaddr_in serv_addr; - uint16_t port; - char user[MAX_USERNAME_SIZE]; - char password[MAX_PASSWORD_SIZE]; - char resultFile[MAX_FILE_NAME_LEN]; - bool use_metric; - bool insert_only; - bool do_aggreFunc; - bool asyncMode; + uint16_t port; + char user[MAX_USERNAME_SIZE]; + char password[MAX_PASSWORD_SIZE]; + char resultFile[MAX_FILE_NAME_LEN]; + bool use_metric; + bool insert_only; + bool do_aggreFunc; + bool asyncMode; - uint32_t threadCount; - uint32_t threadCountByCreateTbl; - uint32_t dbCount; - SDataBase db[MAX_DB_COUNT]; + uint32_t threadCount; + uint32_t threadCountByCreateTbl; + uint32_t dbCount; + SDataBase db[MAX_DB_COUNT]; - // statistics - uint64_t totalInsertRows; - uint64_t totalAffectedRows; + // statistics + uint64_t totalInsertRows; + uint64_t totalAffectedRows; } SDbs; typedef struct SpecifiedQueryInfo_S { - uint64_t queryInterval; // 0: unlimit > 0 loop/s - uint32_t concurrent; - int sqlCount; - uint32_t asyncMode; // 0: sync, 1: async - uint64_t subscribeInterval; // ms - uint64_t queryTimes; - bool subscribeRestart; - int subscribeKeepProgress; - char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; - char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; - int resubAfterConsume[MAX_QUERY_SQL_COUNT]; - int endAfterConsume[MAX_QUERY_SQL_COUNT]; - TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; - char topic[MAX_QUERY_SQL_COUNT][32]; - int consumed[MAX_QUERY_SQL_COUNT]; - TAOS_RES* res[MAX_QUERY_SQL_COUNT]; - uint64_t totalQueried; + uint64_t queryInterval; // 0: unlimit > 0 loop/s + uint32_t concurrent; + int sqlCount; + uint32_t asyncMode; // 0: sync, 1: async + uint64_t subscribeInterval; // ms + uint64_t queryTimes; + bool subscribeRestart; + int subscribeKeepProgress; + char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; + int resubAfterConsume[MAX_QUERY_SQL_COUNT]; + int endAfterConsume[MAX_QUERY_SQL_COUNT]; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; + char topic[MAX_QUERY_SQL_COUNT][32]; + int consumed[MAX_QUERY_SQL_COUNT]; + TAOS_RES* res[MAX_QUERY_SQL_COUNT]; + uint64_t totalQueried; } SpecifiedQueryInfo; typedef struct SuperQueryInfo_S { - char sTblName[TSDB_TABLE_NAME_LEN]; - uint64_t queryInterval; // 0: unlimit > 0 loop/s - uint32_t threadCnt; - uint32_t asyncMode; // 0: sync, 1: async - uint64_t subscribeInterval; // ms - bool subscribeRestart; - int subscribeKeepProgress; - uint64_t queryTimes; - int64_t childTblCount; - char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq - int sqlCount; - char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; - char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; - int resubAfterConsume; - int endAfterConsume; - TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; + char sTblName[TSDB_TABLE_NAME_LEN]; + uint64_t queryInterval; // 0: unlimit > 0 loop/s + uint32_t threadCnt; + uint32_t asyncMode; // 0: sync, 1: async + uint64_t subscribeInterval; // ms + bool subscribeRestart; + int subscribeKeepProgress; + uint64_t queryTimes; + int64_t childTblCount; + char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq + int sqlCount; + char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; + int resubAfterConsume; + int endAfterConsume; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; - char* childTblName; - uint64_t totalQueried; + char* childTblName; + uint64_t totalQueried; } SuperQueryInfo; typedef struct SQueryMetaInfo_S { - char cfgDir[MAX_FILE_NAME_LEN]; - char host[MAX_HOSTNAME_SIZE]; - uint16_t port; - struct sockaddr_in serv_addr; - char user[MAX_USERNAME_SIZE]; - char password[MAX_PASSWORD_SIZE]; - char dbName[TSDB_DB_NAME_LEN]; - char queryMode[MAX_TB_NAME_SIZE]; // taosc, rest + char cfgDir[MAX_FILE_NAME_LEN]; + char host[MAX_HOSTNAME_SIZE]; + uint16_t port; + struct sockaddr_in serv_addr; + char user[MAX_USERNAME_SIZE]; + char password[MAX_PASSWORD_SIZE]; + char dbName[TSDB_DB_NAME_LEN]; + char queryMode[MAX_TB_NAME_SIZE]; // taosc, rest - SpecifiedQueryInfo specifiedQueryInfo; - SuperQueryInfo superQueryInfo; - uint64_t totalQueried; + SpecifiedQueryInfo specifiedQueryInfo; + SuperQueryInfo superQueryInfo; + uint64_t totalQueried; } SQueryMetaInfo; typedef struct SThreadInfo_S { @@ -1140,7 +1141,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { if (arguments->disorderRatio) { printf("# Data order: %d\n", arguments->disorderRatio); printf("# Data out of order rate: %d\n", arguments->disorderRange); - } printf("# Delete method: %d\n", arguments->method_of_delete); printf("# Answer yes when prompt: %d\n", arguments->answer_yes); @@ -1513,7 +1513,10 @@ static int printfInsertMeta() { } if (g_Dbs.db[i].dbCfg.precision[0] != 0) { if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) - || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { +#if NANO_SECOND_ENABLED == 1 + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2)) +#endif + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ns", 2))) { printf(" precision: \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision); } else { @@ -1703,6 +1706,9 @@ static void printfInsertMetaToFile(FILE* fp) { } if (g_Dbs.db[i].dbCfg.precision[0] != 0) { if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) +#if NANO_SECOND_ENABLED == 1 + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ns", 2)) +#endif || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { fprintf(fp, " precision: %s\n", g_Dbs.db[i].dbCfg.precision); @@ -1903,10 +1909,12 @@ static void printfQueryMeta() { static char* formatTimestamp(char* buf, int64_t val, int precision) { time_t tt; - if (precision == TSDB_TIME_PRECISION_NANO) { - tt = (time_t)(val / 1000000000); - } else if (precision == TSDB_TIME_PRECISION_MICRO) { + if (precision == TSDB_TIME_PRECISION_MICRO) { tt = (time_t)(val / 1000000); +#if NANO_SECOND_ENABLED == 1 + } if (precision == TSDB_TIME_PRECISION_NANO) { + tt = (time_t)(val / 1000000000); +#endif } else { tt = (time_t)(val / 1000); } @@ -1926,10 +1934,12 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { struct tm* ptm = localtime(&tt); size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm); - if (precision == TSDB_TIME_PRECISION_NANO) { - sprintf(buf + pos, ".%09d", (int)(val % 1000000000)); - } else if (precision == TSDB_TIME_PRECISION_MICRO) { + if (precision == TSDB_TIME_PRECISION_MICRO) { sprintf(buf + pos, ".%06d", (int)(val % 1000000)); +#if NANO_SECOND_ENABLED == 1 + } else if (precision == TSDB_TIME_PRECISION_NANO) { + sprintf(buf + pos, ".%09d", (int)(val % 1000000000)); +#endif } else { sprintf(buf + pos, ".%03d", (int)(val % 1000)); } @@ -2185,7 +2195,6 @@ static void printfQuerySystemInfo(TAOS * taos) { snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name); res = taos_query(taos, buffer); xDumpResultToFile(filename, res); - free(dbInfos[i]); } @@ -2806,7 +2815,7 @@ static int createSuperTable( if (strcasecmp(dataType, "BINARY") == 0) { if ((g_args.demo_mode) && (tagIndex == 1)) { len += snprintf(tags + len, STRING_LEN - len, - "loction BINARY(%d), ", + "location BINARY(%d), ", superTbl->tags[tagIndex].dataLen); } else { len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", @@ -2875,134 +2884,138 @@ static int createSuperTable( } static int createDatabasesAndStables() { - TAOS * taos = NULL; - int ret = 0; - taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port); - if (taos == NULL) { - errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); - return -1; - } - char command[BUFFER_SIZE] = "\0"; - - for (int i = 0; i < g_Dbs.dbCount; i++) { - if (g_Dbs.db[i].drop) { - sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName); - if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { - taos_close(taos); + TAOS * taos = NULL; + int ret = 0; + taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port); + if (taos == NULL) { + errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); return -1; - } - - int dataLen = 0; - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName); - - if (g_Dbs.db[i].dbCfg.blocks > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks); - } - if (g_Dbs.db[i].dbCfg.cache > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache); - } - if (g_Dbs.db[i].dbCfg.days > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days); - } - if (g_Dbs.db[i].dbCfg.keep > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep); - } - if (g_Dbs.db[i].dbCfg.quorum > 1) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum); - } - if (g_Dbs.db[i].dbCfg.replica > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica); - } - if (g_Dbs.db[i].dbCfg.update > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update); - } - //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) { - // dataLen += snprintf(command + dataLen, - // BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode); - //} - if (g_Dbs.db[i].dbCfg.minRows > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows); - } - if (g_Dbs.db[i].dbCfg.maxRows > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows); - } - if (g_Dbs.db[i].dbCfg.comp > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp); - } - if (g_Dbs.db[i].dbCfg.walLevel > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel); - } - if (g_Dbs.db[i].dbCfg.cacheLast > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast); - } - if (g_Dbs.db[i].dbCfg.fsync > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, - " fsync %d", g_Dbs.db[i].dbCfg.fsync); - } - if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms"))) - || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, - "us", strlen("us")))) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, - " precision \'%s\';", g_Dbs.db[i].dbCfg.precision); - } - - if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { - taos_close(taos); - errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName); - return -1; - } - printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName); } + char command[BUFFER_SIZE] = "\0"; - debugPrint("%s() LN%d supertbl count:%"PRIu64"\n", - __func__, __LINE__, g_Dbs.db[i].superTblCount); + for (int i = 0; i < g_Dbs.dbCount; i++) { + if (g_Dbs.db[i].drop) { + sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName); + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { + taos_close(taos); + return -1; + } - int validStbCount = 0; + int dataLen = 0; + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName); - for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { - sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName, - g_Dbs.db[i].superTbls[j].sTblName); - ret = queryDbExec(taos, command, NO_INSERT_TYPE, true); + if (g_Dbs.db[i].dbCfg.blocks > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks); + } + if (g_Dbs.db[i].dbCfg.cache > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache); + } + if (g_Dbs.db[i].dbCfg.days > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days); + } + if (g_Dbs.db[i].dbCfg.keep > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep); + } + if (g_Dbs.db[i].dbCfg.quorum > 1) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum); + } + if (g_Dbs.db[i].dbCfg.replica > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica); + } + if (g_Dbs.db[i].dbCfg.update > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update); + } + //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) { + // dataLen += snprintf(command + dataLen, + // BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode); + //} + if (g_Dbs.db[i].dbCfg.minRows > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows); + } + if (g_Dbs.db[i].dbCfg.maxRows > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows); + } + if (g_Dbs.db[i].dbCfg.comp > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp); + } + if (g_Dbs.db[i].dbCfg.walLevel > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel); + } + if (g_Dbs.db[i].dbCfg.cacheLast > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast); + } + if (g_Dbs.db[i].dbCfg.fsync > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, + " fsync %d", g_Dbs.db[i].dbCfg.fsync); + } + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms"))) +#if NANO_SECOND_ENABLED == 1 + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, + "ns", strlen("ns"))) +#endif + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, + "us", strlen("us")))) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, + " precision \'%s\';", g_Dbs.db[i].dbCfg.precision); + } - if ((ret != 0) || (g_Dbs.db[i].drop)) { - ret = createSuperTable(taos, g_Dbs.db[i].dbName, - &g_Dbs.db[i].superTbls[j]); - - if (0 != ret) { - errorPrint("create super table %"PRIu64" failed!\n\n", j); - continue; + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { + taos_close(taos); + errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName); + return -1; + } + printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName); } - } - ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName, - &g_Dbs.db[i].superTbls[j]); - if (0 != ret) { - errorPrint("\nget super table %s.%s info failed!\n\n", - g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); - continue; - } + debugPrint("%s() LN%d supertbl count:%"PRIu64"\n", + __func__, __LINE__, g_Dbs.db[i].superTblCount); - validStbCount ++; + int validStbCount = 0; + + for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { + sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName, + g_Dbs.db[i].superTbls[j].sTblName); + ret = queryDbExec(taos, command, NO_INSERT_TYPE, true); + + if ((ret != 0) || (g_Dbs.db[i].drop)) { + ret = createSuperTable(taos, g_Dbs.db[i].dbName, + &g_Dbs.db[i].superTbls[j]); + + if (0 != ret) { + errorPrint("create super table %"PRIu64" failed!\n\n", j); + continue; + } + } + + ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName, + &g_Dbs.db[i].superTbls[j]); + if (0 != ret) { + errorPrint("\nget super table %s.%s info failed!\n\n", + g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); + continue; + } + + validStbCount ++; + } + + g_Dbs.db[i].superTblCount = validStbCount; } - g_Dbs.db[i].superTblCount = validStbCount; - } - - taos_close(taos); - return 0; + taos_close(taos); + return 0; } static void* createTable(void *sarg) @@ -3530,733 +3543,733 @@ PARSE_OVER: } static bool getMetaFromInsertJsonFile(cJSON* root) { - bool ret = false; + bool ret = false; - cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir"); - if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) { - tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN); - } - - cJSON* host = cJSON_GetObjectItem(root, "host"); - if (host && host->type == cJSON_String && host->valuestring != NULL) { - tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE); - } else if (!host) { - tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); - } else { - printf("ERROR: failed to read json, host not found\n"); - goto PARSE_OVER; - } - - cJSON* port = cJSON_GetObjectItem(root, "port"); - if (port && port->type == cJSON_Number) { - g_Dbs.port = port->valueint; - } else if (!port) { - g_Dbs.port = 6030; - } - - cJSON* user = cJSON_GetObjectItem(root, "user"); - if (user && user->type == cJSON_String && user->valuestring != NULL) { - tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE); - } else if (!user) { - tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE); - } - - cJSON* password = cJSON_GetObjectItem(root, "password"); - if (password && password->type == cJSON_String && password->valuestring != NULL) { - tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE); - } else if (!password) { - tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE); - } - - cJSON* resultfile = cJSON_GetObjectItem(root, "result_file"); - if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) { - tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN); - } else if (!resultfile) { - tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN); - } - - cJSON* threads = cJSON_GetObjectItem(root, "thread_count"); - if (threads && threads->type == cJSON_Number) { - g_Dbs.threadCount = threads->valueint; - } else if (!threads) { - g_Dbs.threadCount = 1; - } else { - printf("ERROR: failed to read json, threads not found\n"); - goto PARSE_OVER; - } - - cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl"); - if (threads2 && threads2->type == cJSON_Number) { - g_Dbs.threadCountByCreateTbl = threads2->valueint; - } else if (!threads2) { - g_Dbs.threadCountByCreateTbl = 1; - } else { - errorPrint("%s() LN%d, failed to read json, threads2 not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval"); - if (gInsertInterval && gInsertInterval->type == cJSON_Number) { - if (gInsertInterval->valueint <0) { - errorPrint("%s() LN%d, failed to read json, insert interval input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; + cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir"); + if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) { + tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN); } - g_args.insert_interval = gInsertInterval->valueint; - } else if (!gInsertInterval) { - g_args.insert_interval = 0; - } else { - errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows"); - if (interlaceRows && interlaceRows->type == cJSON_Number) { - if (interlaceRows->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - - } - g_args.interlace_rows = interlaceRows->valueint; - } else if (!interlaceRows) { - g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req - } else { - errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len"); - if (maxSqlLen && maxSqlLen->type == cJSON_Number) { - if (maxSqlLen->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - g_args.max_sql_len = maxSqlLen->valueint; - } else if (!maxSqlLen) { - g_args.max_sql_len = (1024*1024); - } else { - errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req"); - if (numRecPerReq && numRecPerReq->type == cJSON_Number) { - if (numRecPerReq->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) { - printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n", - numRecPerReq->valueint, MAX_RECORDS_PER_REQ); - printf(" number of records per request value will be set to %d\n\n", - MAX_RECORDS_PER_REQ); - prompt(); - numRecPerReq->valueint = MAX_RECORDS_PER_REQ; - } - g_args.num_of_RPR = numRecPerReq->valueint; - } else if (!numRecPerReq) { - g_args.num_of_RPR = MAX_RECORDS_PER_REQ; - } else { - errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, - if (answerPrompt - && answerPrompt->type == cJSON_String - && answerPrompt->valuestring != NULL) { - if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) { - g_args.answer_yes = false; - } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) { - g_args.answer_yes = true; + cJSON* host = cJSON_GetObjectItem(root, "host"); + if (host && host->type == cJSON_String && host->valuestring != NULL) { + tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE); + } else if (!host) { + tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); } else { - g_args.answer_yes = false; - } - } else if (!answerPrompt) { - g_args.answer_yes = true; // default is no, mean answer_yes. - } else { - errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n"); - goto PARSE_OVER; - } - - // rows per table need be less than insert batch - if (g_args.interlace_rows > g_args.num_of_RPR) { - printf("NOTICE: interlace rows value %u > num_of_records_per_req %u\n\n", - g_args.interlace_rows, g_args.num_of_RPR); - printf(" interlace rows value will be set to num_of_records_per_req %u\n\n", - g_args.num_of_RPR); - prompt(); - g_args.interlace_rows = g_args.num_of_RPR; - } - - cJSON* dbs = cJSON_GetObjectItem(root, "databases"); - if (!dbs || dbs->type != cJSON_Array) { - printf("ERROR: failed to read json, databases not found\n"); - goto PARSE_OVER; - } - - int dbSize = cJSON_GetArraySize(dbs); - if (dbSize > MAX_DB_COUNT) { - errorPrint( - "ERROR: failed to read json, databases size overflow, max database is %d\n", - MAX_DB_COUNT); - goto PARSE_OVER; - } - - g_Dbs.dbCount = dbSize; - for (int i = 0; i < dbSize; ++i) { - cJSON* dbinfos = cJSON_GetArrayItem(dbs, i); - if (dbinfos == NULL) continue; - - // dbinfo - cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo"); - if (!dbinfo || dbinfo->type != cJSON_Object) { - printf("ERROR: failed to read json, dbinfo not found\n"); - goto PARSE_OVER; + printf("ERROR: failed to read json, host not found\n"); + goto PARSE_OVER; } - cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name"); - if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) { - printf("ERROR: failed to read json, db name not found\n"); - goto PARSE_OVER; + cJSON* port = cJSON_GetObjectItem(root, "port"); + if (port && port->type == cJSON_Number) { + g_Dbs.port = port->valueint; + } else if (!port) { + g_Dbs.port = 6030; } - tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, TSDB_DB_NAME_LEN); - cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop"); - if (drop && drop->type == cJSON_String && drop->valuestring != NULL) { - if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) { - g_Dbs.db[i].drop = true; - } else { - g_Dbs.db[i].drop = false; - } - } else if (!drop) { - g_Dbs.db[i].drop = g_args.drop_database; + cJSON* user = cJSON_GetObjectItem(root, "user"); + if (user && user->type == cJSON_String && user->valuestring != NULL) { + tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE); + } else if (!user) { + tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE); + } + + cJSON* password = cJSON_GetObjectItem(root, "password"); + if (password && password->type == cJSON_String && password->valuestring != NULL) { + tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE); + } else if (!password) { + tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE); + } + + cJSON* resultfile = cJSON_GetObjectItem(root, "result_file"); + if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) { + tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN); + } else if (!resultfile) { + tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN); + } + + cJSON* threads = cJSON_GetObjectItem(root, "thread_count"); + if (threads && threads->type == cJSON_Number) { + g_Dbs.threadCount = threads->valueint; + } else if (!threads) { + g_Dbs.threadCount = 1; } else { - errorPrint("%s() LN%d, failed to read json, drop input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; + printf("ERROR: failed to read json, threads not found\n"); + goto PARSE_OVER; } - cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision"); - if (precision && precision->type == cJSON_String - && precision->valuestring != NULL) { - tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring, - 8); - } else if (!precision) { - memset(g_Dbs.db[i].dbCfg.precision, 0, 8); + cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl"); + if (threads2 && threads2->type == cJSON_Number) { + g_Dbs.threadCountByCreateTbl = threads2->valueint; + } else if (!threads2) { + g_Dbs.threadCountByCreateTbl = 1; } else { - printf("ERROR: failed to read json, precision not found\n"); - goto PARSE_OVER; - } - - cJSON* update = cJSON_GetObjectItem(dbinfo, "update"); - if (update && update->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.update = update->valueint; - } else if (!update) { - g_Dbs.db[i].dbCfg.update = -1; - } else { - printf("ERROR: failed to read json, update not found\n"); - goto PARSE_OVER; - } - - cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica"); - if (replica && replica->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.replica = replica->valueint; - } else if (!replica) { - g_Dbs.db[i].dbCfg.replica = -1; - } else { - printf("ERROR: failed to read json, replica not found\n"); - goto PARSE_OVER; - } - - cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep"); - if (keep && keep->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.keep = keep->valueint; - } else if (!keep) { - g_Dbs.db[i].dbCfg.keep = -1; - } else { - printf("ERROR: failed to read json, keep not found\n"); - goto PARSE_OVER; - } - - cJSON* days = cJSON_GetObjectItem(dbinfo, "days"); - if (days && days->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.days = days->valueint; - } else if (!days) { - g_Dbs.db[i].dbCfg.days = -1; - } else { - printf("ERROR: failed to read json, days not found\n"); - goto PARSE_OVER; - } - - cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache"); - if (cache && cache->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.cache = cache->valueint; - } else if (!cache) { - g_Dbs.db[i].dbCfg.cache = -1; - } else { - printf("ERROR: failed to read json, cache not found\n"); - goto PARSE_OVER; - } - - cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks"); - if (blocks && blocks->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.blocks = blocks->valueint; - } else if (!blocks) { - g_Dbs.db[i].dbCfg.blocks = -1; - } else { - printf("ERROR: failed to read json, block not found\n"); - goto PARSE_OVER; - } - - //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode"); - //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) { - // g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint; - //} else if (!maxtablesPerVnode) { - // g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES; - //} else { - // printf("failed to read json, maxtablesPerVnode not found"); - // goto PARSE_OVER; - //} - - cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows"); - if (minRows && minRows->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.minRows = minRows->valueint; - } else if (!minRows) { - g_Dbs.db[i].dbCfg.minRows = 0; // 0 means default - } else { - printf("ERROR: failed to read json, minRows not found\n"); - goto PARSE_OVER; - } - - cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows"); - if (maxRows && maxRows->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint; - } else if (!maxRows) { - g_Dbs.db[i].dbCfg.maxRows = 0; // 0 means default - } else { - printf("ERROR: failed to read json, maxRows not found\n"); - goto PARSE_OVER; - } - - cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp"); - if (comp && comp->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.comp = comp->valueint; - } else if (!comp) { - g_Dbs.db[i].dbCfg.comp = -1; - } else { - printf("ERROR: failed to read json, comp not found\n"); - goto PARSE_OVER; - } - - cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel"); - if (walLevel && walLevel->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint; - } else if (!walLevel) { - g_Dbs.db[i].dbCfg.walLevel = -1; - } else { - printf("ERROR: failed to read json, walLevel not found\n"); - goto PARSE_OVER; - } - - cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast"); - if (cacheLast && cacheLast->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint; - } else if (!cacheLast) { - g_Dbs.db[i].dbCfg.cacheLast = -1; - } else { - printf("ERROR: failed to read json, cacheLast not found\n"); - goto PARSE_OVER; - } - - cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum"); - if (quorum && quorum->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.quorum = quorum->valueint; - } else if (!quorum) { - g_Dbs.db[i].dbCfg.quorum = 1; - } else { - printf("failed to read json, quorum input mistake"); - goto PARSE_OVER; - } - - cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync"); - if (fsync && fsync->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.fsync = fsync->valueint; - } else if (!fsync) { - g_Dbs.db[i].dbCfg.fsync = -1; - } else { - errorPrint("%s() LN%d, failed to read json, fsync input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - // super_talbes - cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables"); - if (!stables || stables->type != cJSON_Array) { - errorPrint("%s() LN%d, failed to read json, super_tables not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - int stbSize = cJSON_GetArraySize(stables); - if (stbSize > MAX_SUPER_TABLE_COUNT) { - errorPrint( - "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n", - __func__, __LINE__, MAX_SUPER_TABLE_COUNT); - goto PARSE_OVER; - } - - g_Dbs.db[i].superTblCount = stbSize; - for (int j = 0; j < stbSize; ++j) { - cJSON* stbInfo = cJSON_GetArrayItem(stables, j); - if (stbInfo == NULL) continue; - - // dbinfo - cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name"); - if (!stbName || stbName->type != cJSON_String - || stbName->valuestring == NULL) { - errorPrint("%s() LN%d, failed to read json, stb name not found\n", + errorPrint("%s() LN%d, failed to read json, threads2 not found\n", __func__, __LINE__); goto PARSE_OVER; - } - tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, - TSDB_TABLE_NAME_LEN); + } - cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix"); - if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) { - printf("ERROR: failed to read json, childtable_prefix not found\n"); - goto PARSE_OVER; - } - tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, - TSDB_TABLE_NAME_LEN - 20); - - cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); - if (autoCreateTbl - && autoCreateTbl->type == cJSON_String - && autoCreateTbl->valuestring != NULL) { - if ((0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) - && (TBL_ALREADY_EXISTS != g_Dbs.db[i].superTbls[j].childTblExists)) { - g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL; - } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) { - g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; - } else { - g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; - } - } else if (!autoCreateTbl) { - g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; - } else { - printf("ERROR: failed to read json, auto_create_table not found\n"); - goto PARSE_OVER; - } - - cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num"); - if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint; - } else if (!batchCreateTbl) { - g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000; - } else { - printf("ERROR: failed to read json, batch_create_tbl_num not found\n"); - goto PARSE_OVER; - } - - cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no - if (childTblExists - && childTblExists->type == cJSON_String - && childTblExists->valuestring != NULL) { - if ((0 == strncasecmp(childTblExists->valuestring, "yes", 3)) - && (g_Dbs.db[i].drop == false)) { - g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS; - } else if ((0 == strncasecmp(childTblExists->valuestring, "no", 2) - || (g_Dbs.db[i].drop == true))) { - g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; - } else { - g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; - } - } else if (!childTblExists) { - g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; - } else { - errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { - g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; - } - - cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count"); - if (!count || count->type != cJSON_Number || 0 >= count->valueint) { - errorPrint("%s() LN%d, failed to read json, childtable_count input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - g_Dbs.db[i].superTbls[j].childTblCount = count->valueint; - - cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source"); - if (dataSource && dataSource->type == cJSON_String - && dataSource->valuestring != NULL) { - tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, - dataSource->valuestring, TSDB_DB_NAME_LEN); - } else if (!dataSource) { - tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", TSDB_DB_NAME_LEN); - } else { - errorPrint("%s() LN%d, failed to read json, data_source not found\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - cJSON *stbIface = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest, stmt - if (stbIface && stbIface->type == cJSON_String - && stbIface->valuestring != NULL) { - if (0 == strcasecmp(stbIface->valuestring, "taosc")) { - g_Dbs.db[i].superTbls[j].iface= TAOSC_IFACE; - } else if (0 == strcasecmp(stbIface->valuestring, "rest")) { - g_Dbs.db[i].superTbls[j].iface= REST_IFACE; -#if STMT_IFACE_ENABLED == 1 - } else if (0 == strcasecmp(stbIface->valuestring, "stmt")) { - g_Dbs.db[i].superTbls[j].iface= STMT_IFACE; -#endif - } else { - errorPrint("%s() LN%d, failed to read json, insert_mode %s not recognized\n", - __func__, __LINE__, stbIface->valuestring); + cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval"); + if (gInsertInterval && gInsertInterval->type == cJSON_Number) { + if (gInsertInterval->valueint <0) { + errorPrint("%s() LN%d, failed to read json, insert interval input mistake\n", + __func__, __LINE__); goto PARSE_OVER; } - } else if (!stbIface) { - g_Dbs.db[i].superTbls[j].iface = TAOSC_IFACE; - } else { - errorPrint("%s", "failed to read json, insert_mode not found\n"); - goto PARSE_OVER; - } - - cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit"); - if ((childTbl_limit) && (g_Dbs.db[i].drop != true) - && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) { - if (childTbl_limit->type != cJSON_Number) { - printf("ERROR: failed to read json, childtable_limit\n"); - goto PARSE_OVER; - } - g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint; - } else { - g_Dbs.db[i].superTbls[j].childTblLimit = -1; // select ... limit -1 means all query result, drop = yes mean all table need recreate, limit value is invalid. - } - - cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset"); - if ((childTbl_offset) && (g_Dbs.db[i].drop != true) - && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) { - if ((childTbl_offset->type != cJSON_Number) - || (0 > childTbl_offset->valueint)) { - printf("ERROR: failed to read json, childtable_offset\n"); - goto PARSE_OVER; - } - g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint; - } else { - g_Dbs.db[i].superTbls[j].childTblOffset = 0; - } - - cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp"); - if (ts && ts->type == cJSON_String && ts->valuestring != NULL) { - tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, - ts->valuestring, TSDB_DB_NAME_LEN); - } else if (!ts) { - tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, - "now", TSDB_DB_NAME_LEN); - } else { - printf("ERROR: failed to read json, start_timestamp not found\n"); - goto PARSE_OVER; - } - - cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step"); - if (timestampStep && timestampStep->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint; - } else if (!timestampStep) { - g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP; - } else { - printf("ERROR: failed to read json, timestamp_step not found\n"); - goto PARSE_OVER; - } - - cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format"); - if (sampleFormat && sampleFormat->type - == cJSON_String && sampleFormat->valuestring != NULL) { - tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, - sampleFormat->valuestring, TSDB_DB_NAME_LEN); - } else if (!sampleFormat) { - tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", TSDB_DB_NAME_LEN); - } else { - printf("ERROR: failed to read json, sample_format not found\n"); - goto PARSE_OVER; - } - - cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file"); - if (sampleFile && sampleFile->type == cJSON_String - && sampleFile->valuestring != NULL) { - tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile, - sampleFile->valuestring, MAX_FILE_NAME_LEN); - } else if (!sampleFile) { - memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN); - } else { - printf("ERROR: failed to read json, sample_file not found\n"); - goto PARSE_OVER; - } - - cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file"); - if ((tagsFile && tagsFile->type == cJSON_String) - && (tagsFile->valuestring != NULL)) { - tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile, - tagsFile->valuestring, MAX_FILE_NAME_LEN); - if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) { - g_Dbs.db[i].superTbls[j].tagSource = 0; - } else { - g_Dbs.db[i].superTbls[j].tagSource = 1; - } - } else if (!tagsFile) { - memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN); - g_Dbs.db[i].superTbls[j].tagSource = 0; - } else { - printf("ERROR: failed to read json, tags_file not found\n"); - goto PARSE_OVER; - } - - cJSON* stbMaxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len"); - if (stbMaxSqlLen && stbMaxSqlLen->type == cJSON_Number) { - int32_t len = stbMaxSqlLen->valueint; - if (len > TSDB_MAX_ALLOWED_SQL_LEN) { - len = TSDB_MAX_ALLOWED_SQL_LEN; - } else if (len < 5) { - len = 5; - } - g_Dbs.db[i].superTbls[j].maxSqlLen = len; - } else if (!maxSqlLen) { - g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len; - } else { - errorPrint("%s() LN%d, failed to read json, stbMaxSqlLen input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } -/* - cJSON *multiThreadWriteOneTbl = - cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes - if (multiThreadWriteOneTbl - && multiThreadWriteOneTbl->type == cJSON_String - && multiThreadWriteOneTbl->valuestring != NULL) { - if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) { - g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1; - } else { - g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; - } - } else if (!multiThreadWriteOneTbl) { - g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; - } else { - printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n"); - goto PARSE_OVER; - } -*/ - cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows"); - if (insertRows && insertRows->type == cJSON_Number) { - if (insertRows->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint; - } else if (!insertRows) { - g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; - } else { - errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - cJSON* stbInterlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows"); - if (stbInterlaceRows && stbInterlaceRows->type == cJSON_Number) { - if (stbInterlaceRows->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - g_Dbs.db[i].superTbls[j].interlaceRows = stbInterlaceRows->valueint; - - if (g_Dbs.db[i].superTbls[j].interlaceRows > g_Dbs.db[i].superTbls[j].insertRows) { - printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %u > insert_rows %"PRId64"\n\n", - i, j, g_Dbs.db[i].superTbls[j].interlaceRows, - g_Dbs.db[i].superTbls[j].insertRows); - printf(" interlace rows value will be set to insert_rows %"PRId64"\n\n", - g_Dbs.db[i].superTbls[j].insertRows); - prompt(); - g_Dbs.db[i].superTbls[j].interlaceRows = g_Dbs.db[i].superTbls[j].insertRows; - } - } else if (!stbInterlaceRows) { - g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req - } else { - errorPrint( - "%s() LN%d, failed to read json, interlace rows input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - - cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio"); - if (disorderRatio && disorderRatio->type == cJSON_Number) { - if (disorderRatio->valueint > 50) - disorderRatio->valueint = 50; - - if (disorderRatio->valueint < 0) - disorderRatio->valueint = 0; - - g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint; - } else if (!disorderRatio) { - g_Dbs.db[i].superTbls[j].disorderRatio = 0; - } else { - printf("ERROR: failed to read json, disorderRatio not found\n"); - goto PARSE_OVER; - } - - cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range"); - if (disorderRange && disorderRange->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint; - } else if (!disorderRange) { - g_Dbs.db[i].superTbls[j].disorderRange = 1000; - } else { - printf("ERROR: failed to read json, disorderRange not found\n"); - goto PARSE_OVER; - } - - cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval"); - if (insertInterval && insertInterval->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint; - if (insertInterval->valueint < 0) { - errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", - __func__, __LINE__); - goto PARSE_OVER; - } - } else if (!insertInterval) { - verbosePrint("%s() LN%d: stable insert interval be overrided by global %"PRIu64".\n", - __func__, __LINE__, g_args.insert_interval); - g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval; - } else { + g_args.insert_interval = gInsertInterval->valueint; + } else if (!gInsertInterval) { + g_args.insert_interval = 0; + } else { errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", __func__, __LINE__); goto PARSE_OVER; - } - - int retVal = getColumnAndTagTypeFromInsertJsonFile( - stbInfo, &g_Dbs.db[i].superTbls[j]); - if (false == retVal) { - goto PARSE_OVER; - } } - } - ret = true; + cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows"); + if (interlaceRows && interlaceRows->type == cJSON_Number) { + if (interlaceRows->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + + } + g_args.interlace_rows = interlaceRows->valueint; + } else if (!interlaceRows) { + g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req + } else { + errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len"); + if (maxSqlLen && maxSqlLen->type == cJSON_Number) { + if (maxSqlLen->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + g_args.max_sql_len = maxSqlLen->valueint; + } else if (!maxSqlLen) { + g_args.max_sql_len = (1024*1024); + } else { + errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req"); + if (numRecPerReq && numRecPerReq->type == cJSON_Number) { + if (numRecPerReq->valueint <= 0) { + errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) { + printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n", + numRecPerReq->valueint, MAX_RECORDS_PER_REQ); + printf(" number of records per request value will be set to %d\n\n", + MAX_RECORDS_PER_REQ); + prompt(); + numRecPerReq->valueint = MAX_RECORDS_PER_REQ; + } + g_args.num_of_RPR = numRecPerReq->valueint; + } else if (!numRecPerReq) { + g_args.num_of_RPR = MAX_RECORDS_PER_REQ; + } else { + errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, + if (answerPrompt + && answerPrompt->type == cJSON_String + && answerPrompt->valuestring != NULL) { + if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) { + g_args.answer_yes = false; + } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) { + g_args.answer_yes = true; + } else { + g_args.answer_yes = false; + } + } else if (!answerPrompt) { + g_args.answer_yes = true; // default is no, mean answer_yes. + } else { + errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n"); + goto PARSE_OVER; + } + + // rows per table need be less than insert batch + if (g_args.interlace_rows > g_args.num_of_RPR) { + printf("NOTICE: interlace rows value %u > num_of_records_per_req %u\n\n", + g_args.interlace_rows, g_args.num_of_RPR); + printf(" interlace rows value will be set to num_of_records_per_req %u\n\n", + g_args.num_of_RPR); + prompt(); + g_args.interlace_rows = g_args.num_of_RPR; + } + + cJSON* dbs = cJSON_GetObjectItem(root, "databases"); + if (!dbs || dbs->type != cJSON_Array) { + printf("ERROR: failed to read json, databases not found\n"); + goto PARSE_OVER; + } + + int dbSize = cJSON_GetArraySize(dbs); + if (dbSize > MAX_DB_COUNT) { + errorPrint( + "ERROR: failed to read json, databases size overflow, max database is %d\n", + MAX_DB_COUNT); + goto PARSE_OVER; + } + + g_Dbs.dbCount = dbSize; + for (int i = 0; i < dbSize; ++i) { + cJSON* dbinfos = cJSON_GetArrayItem(dbs, i); + if (dbinfos == NULL) continue; + + // dbinfo + cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo"); + if (!dbinfo || dbinfo->type != cJSON_Object) { + printf("ERROR: failed to read json, dbinfo not found\n"); + goto PARSE_OVER; + } + + cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name"); + if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) { + printf("ERROR: failed to read json, db name not found\n"); + goto PARSE_OVER; + } + tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, TSDB_DB_NAME_LEN); + + cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop"); + if (drop && drop->type == cJSON_String && drop->valuestring != NULL) { + if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) { + g_Dbs.db[i].drop = true; + } else { + g_Dbs.db[i].drop = false; + } + } else if (!drop) { + g_Dbs.db[i].drop = g_args.drop_database; + } else { + errorPrint("%s() LN%d, failed to read json, drop input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision"); + if (precision && precision->type == cJSON_String + && precision->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring, + 8); + } else if (!precision) { + memset(g_Dbs.db[i].dbCfg.precision, 0, 8); + } else { + printf("ERROR: failed to read json, precision not found\n"); + goto PARSE_OVER; + } + + cJSON* update = cJSON_GetObjectItem(dbinfo, "update"); + if (update && update->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.update = update->valueint; + } else if (!update) { + g_Dbs.db[i].dbCfg.update = -1; + } else { + printf("ERROR: failed to read json, update not found\n"); + goto PARSE_OVER; + } + + cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica"); + if (replica && replica->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.replica = replica->valueint; + } else if (!replica) { + g_Dbs.db[i].dbCfg.replica = -1; + } else { + printf("ERROR: failed to read json, replica not found\n"); + goto PARSE_OVER; + } + + cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep"); + if (keep && keep->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.keep = keep->valueint; + } else if (!keep) { + g_Dbs.db[i].dbCfg.keep = -1; + } else { + printf("ERROR: failed to read json, keep not found\n"); + goto PARSE_OVER; + } + + cJSON* days = cJSON_GetObjectItem(dbinfo, "days"); + if (days && days->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.days = days->valueint; + } else if (!days) { + g_Dbs.db[i].dbCfg.days = -1; + } else { + printf("ERROR: failed to read json, days not found\n"); + goto PARSE_OVER; + } + + cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache"); + if (cache && cache->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.cache = cache->valueint; + } else if (!cache) { + g_Dbs.db[i].dbCfg.cache = -1; + } else { + printf("ERROR: failed to read json, cache not found\n"); + goto PARSE_OVER; + } + + cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks"); + if (blocks && blocks->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.blocks = blocks->valueint; + } else if (!blocks) { + g_Dbs.db[i].dbCfg.blocks = -1; + } else { + printf("ERROR: failed to read json, block not found\n"); + goto PARSE_OVER; + } + + //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode"); + //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) { + // g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint; + //} else if (!maxtablesPerVnode) { + // g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES; + //} else { + // printf("failed to read json, maxtablesPerVnode not found"); + // goto PARSE_OVER; + //} + + cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows"); + if (minRows && minRows->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.minRows = minRows->valueint; + } else if (!minRows) { + g_Dbs.db[i].dbCfg.minRows = 0; // 0 means default + } else { + printf("ERROR: failed to read json, minRows not found\n"); + goto PARSE_OVER; + } + + cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows"); + if (maxRows && maxRows->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint; + } else if (!maxRows) { + g_Dbs.db[i].dbCfg.maxRows = 0; // 0 means default + } else { + printf("ERROR: failed to read json, maxRows not found\n"); + goto PARSE_OVER; + } + + cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp"); + if (comp && comp->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.comp = comp->valueint; + } else if (!comp) { + g_Dbs.db[i].dbCfg.comp = -1; + } else { + printf("ERROR: failed to read json, comp not found\n"); + goto PARSE_OVER; + } + + cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel"); + if (walLevel && walLevel->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint; + } else if (!walLevel) { + g_Dbs.db[i].dbCfg.walLevel = -1; + } else { + printf("ERROR: failed to read json, walLevel not found\n"); + goto PARSE_OVER; + } + + cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast"); + if (cacheLast && cacheLast->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint; + } else if (!cacheLast) { + g_Dbs.db[i].dbCfg.cacheLast = -1; + } else { + printf("ERROR: failed to read json, cacheLast not found\n"); + goto PARSE_OVER; + } + + cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum"); + if (quorum && quorum->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.quorum = quorum->valueint; + } else if (!quorum) { + g_Dbs.db[i].dbCfg.quorum = 1; + } else { + printf("failed to read json, quorum input mistake"); + goto PARSE_OVER; + } + + cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync"); + if (fsync && fsync->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.fsync = fsync->valueint; + } else if (!fsync) { + g_Dbs.db[i].dbCfg.fsync = -1; + } else { + errorPrint("%s() LN%d, failed to read json, fsync input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + // super_talbes + cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables"); + if (!stables || stables->type != cJSON_Array) { + errorPrint("%s() LN%d, failed to read json, super_tables not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + int stbSize = cJSON_GetArraySize(stables); + if (stbSize > MAX_SUPER_TABLE_COUNT) { + errorPrint( + "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n", + __func__, __LINE__, MAX_SUPER_TABLE_COUNT); + goto PARSE_OVER; + } + + g_Dbs.db[i].superTblCount = stbSize; + for (int j = 0; j < stbSize; ++j) { + cJSON* stbInfo = cJSON_GetArrayItem(stables, j); + if (stbInfo == NULL) continue; + + // dbinfo + cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name"); + if (!stbName || stbName->type != cJSON_String + || stbName->valuestring == NULL) { + errorPrint("%s() LN%d, failed to read json, stb name not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, + TSDB_TABLE_NAME_LEN); + + cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix"); + if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) { + printf("ERROR: failed to read json, childtable_prefix not found\n"); + goto PARSE_OVER; + } + tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, + TSDB_TABLE_NAME_LEN - 20); + + cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); + if (autoCreateTbl + && autoCreateTbl->type == cJSON_String + && autoCreateTbl->valuestring != NULL) { + if ((0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) + && (TBL_ALREADY_EXISTS != g_Dbs.db[i].superTbls[j].childTblExists)) { + g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL; + } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } else { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } + } else if (!autoCreateTbl) { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } else { + printf("ERROR: failed to read json, auto_create_table not found\n"); + goto PARSE_OVER; + } + + cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num"); + if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint; + } else if (!batchCreateTbl) { + g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000; + } else { + printf("ERROR: failed to read json, batch_create_tbl_num not found\n"); + goto PARSE_OVER; + } + + cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no + if (childTblExists + && childTblExists->type == cJSON_String + && childTblExists->valuestring != NULL) { + if ((0 == strncasecmp(childTblExists->valuestring, "yes", 3)) + && (g_Dbs.db[i].drop == false)) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS; + } else if ((0 == strncasecmp(childTblExists->valuestring, "no", 2) + || (g_Dbs.db[i].drop == true))) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } else { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } + } else if (!childTblExists) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } else { + errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } + + cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count"); + if (!count || count->type != cJSON_Number || 0 >= count->valueint) { + errorPrint("%s() LN%d, failed to read json, childtable_count input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].childTblCount = count->valueint; + + cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source"); + if (dataSource && dataSource->type == cJSON_String + && dataSource->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, + dataSource->valuestring, TSDB_DB_NAME_LEN); + } else if (!dataSource) { + tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", TSDB_DB_NAME_LEN); + } else { + errorPrint("%s() LN%d, failed to read json, data_source not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON *stbIface = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest, stmt + if (stbIface && stbIface->type == cJSON_String + && stbIface->valuestring != NULL) { + if (0 == strcasecmp(stbIface->valuestring, "taosc")) { + g_Dbs.db[i].superTbls[j].iface= TAOSC_IFACE; + } else if (0 == strcasecmp(stbIface->valuestring, "rest")) { + g_Dbs.db[i].superTbls[j].iface= REST_IFACE; +#if STMT_IFACE_ENABLED == 1 + } else if (0 == strcasecmp(stbIface->valuestring, "stmt")) { + g_Dbs.db[i].superTbls[j].iface= STMT_IFACE; +#endif + } else { + errorPrint("%s() LN%d, failed to read json, insert_mode %s not recognized\n", + __func__, __LINE__, stbIface->valuestring); + goto PARSE_OVER; + } + } else if (!stbIface) { + g_Dbs.db[i].superTbls[j].iface = TAOSC_IFACE; + } else { + errorPrint("%s", "failed to read json, insert_mode not found\n"); + goto PARSE_OVER; + } + + cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit"); + if ((childTbl_limit) && (g_Dbs.db[i].drop != true) + && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) { + if (childTbl_limit->type != cJSON_Number) { + printf("ERROR: failed to read json, childtable_limit\n"); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint; + } else { + g_Dbs.db[i].superTbls[j].childTblLimit = -1; // select ... limit -1 means all query result, drop = yes mean all table need recreate, limit value is invalid. + } + + cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset"); + if ((childTbl_offset) && (g_Dbs.db[i].drop != true) + && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) { + if ((childTbl_offset->type != cJSON_Number) + || (0 > childTbl_offset->valueint)) { + printf("ERROR: failed to read json, childtable_offset\n"); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint; + } else { + g_Dbs.db[i].superTbls[j].childTblOffset = 0; + } + + cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp"); + if (ts && ts->type == cJSON_String && ts->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, + ts->valuestring, TSDB_DB_NAME_LEN); + } else if (!ts) { + tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, + "now", TSDB_DB_NAME_LEN); + } else { + printf("ERROR: failed to read json, start_timestamp not found\n"); + goto PARSE_OVER; + } + + cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step"); + if (timestampStep && timestampStep->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint; + } else if (!timestampStep) { + g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP; + } else { + printf("ERROR: failed to read json, timestamp_step not found\n"); + goto PARSE_OVER; + } + + cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format"); + if (sampleFormat && sampleFormat->type + == cJSON_String && sampleFormat->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, + sampleFormat->valuestring, TSDB_DB_NAME_LEN); + } else if (!sampleFormat) { + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", TSDB_DB_NAME_LEN); + } else { + printf("ERROR: failed to read json, sample_format not found\n"); + goto PARSE_OVER; + } + + cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file"); + if (sampleFile && sampleFile->type == cJSON_String + && sampleFile->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile, + sampleFile->valuestring, MAX_FILE_NAME_LEN); + } else if (!sampleFile) { + memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN); + } else { + printf("ERROR: failed to read json, sample_file not found\n"); + goto PARSE_OVER; + } + + cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file"); + if ((tagsFile && tagsFile->type == cJSON_String) + && (tagsFile->valuestring != NULL)) { + tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile, + tagsFile->valuestring, MAX_FILE_NAME_LEN); + if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) { + g_Dbs.db[i].superTbls[j].tagSource = 0; + } else { + g_Dbs.db[i].superTbls[j].tagSource = 1; + } + } else if (!tagsFile) { + memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN); + g_Dbs.db[i].superTbls[j].tagSource = 0; + } else { + printf("ERROR: failed to read json, tags_file not found\n"); + goto PARSE_OVER; + } + + cJSON* stbMaxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len"); + if (stbMaxSqlLen && stbMaxSqlLen->type == cJSON_Number) { + int32_t len = stbMaxSqlLen->valueint; + if (len > TSDB_MAX_ALLOWED_SQL_LEN) { + len = TSDB_MAX_ALLOWED_SQL_LEN; + } else if (len < 5) { + len = 5; + } + g_Dbs.db[i].superTbls[j].maxSqlLen = len; + } else if (!maxSqlLen) { + g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len; + } else { + errorPrint("%s() LN%d, failed to read json, stbMaxSqlLen input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + /* + cJSON *multiThreadWriteOneTbl = + cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes + if (multiThreadWriteOneTbl + && multiThreadWriteOneTbl->type == cJSON_String + && multiThreadWriteOneTbl->valuestring != NULL) { + if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1; + } else { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; + } + } else if (!multiThreadWriteOneTbl) { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; + } else { + printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n"); + goto PARSE_OVER; + } + */ + cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows"); + if (insertRows && insertRows->type == cJSON_Number) { + if (insertRows->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint; + } else if (!insertRows) { + g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; + } else { + errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* stbInterlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows"); + if (stbInterlaceRows && stbInterlaceRows->type == cJSON_Number) { + if (stbInterlaceRows->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].interlaceRows = stbInterlaceRows->valueint; + + if (g_Dbs.db[i].superTbls[j].interlaceRows > g_Dbs.db[i].superTbls[j].insertRows) { + printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %u > insert_rows %"PRId64"\n\n", + i, j, g_Dbs.db[i].superTbls[j].interlaceRows, + g_Dbs.db[i].superTbls[j].insertRows); + printf(" interlace rows value will be set to insert_rows %"PRId64"\n\n", + g_Dbs.db[i].superTbls[j].insertRows); + prompt(); + g_Dbs.db[i].superTbls[j].interlaceRows = g_Dbs.db[i].superTbls[j].insertRows; + } + } else if (!stbInterlaceRows) { + g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req + } else { + errorPrint( + "%s() LN%d, failed to read json, interlace rows input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio"); + if (disorderRatio && disorderRatio->type == cJSON_Number) { + if (disorderRatio->valueint > 50) + disorderRatio->valueint = 50; + + if (disorderRatio->valueint < 0) + disorderRatio->valueint = 0; + + g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint; + } else if (!disorderRatio) { + g_Dbs.db[i].superTbls[j].disorderRatio = 0; + } else { + printf("ERROR: failed to read json, disorderRatio not found\n"); + goto PARSE_OVER; + } + + cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range"); + if (disorderRange && disorderRange->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint; + } else if (!disorderRange) { + g_Dbs.db[i].superTbls[j].disorderRange = 1000; + } else { + printf("ERROR: failed to read json, disorderRange not found\n"); + goto PARSE_OVER; + } + + cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval"); + if (insertInterval && insertInterval->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint; + if (insertInterval->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + } else if (!insertInterval) { + verbosePrint("%s() LN%d: stable insert interval be overrided by global %"PRIu64".\n", + __func__, __LINE__, g_args.insert_interval); + g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval; + } else { + errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + int retVal = getColumnAndTagTypeFromInsertJsonFile( + stbInfo, &g_Dbs.db[i].superTbls[j]); + if (false == retVal) { + goto PARSE_OVER; + } + } + } + + ret = true; PARSE_OVER: - return ret; + return ret; } static bool getMetaFromQueryJsonFile(cJSON* root) { @@ -4957,62 +4970,64 @@ static int64_t generateStbRowData( static int64_t generateData(char *recBuf, char **data_type, int64_t timestamp, int lenOfBinary) { - memset(recBuf, 0, MAX_DATA_SIZE); - char *pstr = recBuf; - pstr += sprintf(pstr, "(%" PRId64, timestamp); + memset(recBuf, 0, MAX_DATA_SIZE); + char *pstr = recBuf; + pstr += sprintf(pstr, "(%" PRId64, timestamp); - int columnCount = g_args.num_of_CPR; + int columnCount = g_args.num_of_CPR; - for (int i = 0; i < columnCount; i++) { - if (strcasecmp(data_type[i % columnCount], "TINYINT") == 0) { - pstr += sprintf(pstr, ",%d", rand_tinyint() ); - } else if (strcasecmp(data_type[i % columnCount], "SMALLINT") == 0) { - pstr += sprintf(pstr, ",%d", rand_smallint()); - } else if (strcasecmp(data_type[i % columnCount], "INT") == 0) { - pstr += sprintf(pstr, ",%d", rand_int()); - } else if (strcasecmp(data_type[i % columnCount], "BIGINT") == 0) { - pstr += sprintf(pstr, ",%" PRId64, rand_bigint()); - } else if (strcasecmp(data_type[i % columnCount], "TIMESTAMP") == 0) { - pstr += sprintf(pstr, ",%" PRId64, rand_bigint()); - } else if (strcasecmp(data_type[i % columnCount], "FLOAT") == 0) { - pstr += sprintf(pstr, ",%10.4f", rand_float()); - } else if (strcasecmp(data_type[i % columnCount], "DOUBLE") == 0) { - double t = rand_double(); - pstr += sprintf(pstr, ",%20.8f", t); - } else if (strcasecmp(data_type[i % columnCount], "BOOL") == 0) { - bool b = rand_bool() & 1; - pstr += sprintf(pstr, ",%s", b ? "true" : "false"); - } else if (strcasecmp(data_type[i % columnCount], "BINARY") == 0) { - char *s = malloc(lenOfBinary + 1); - if (s == NULL) { - errorPrint("%s() LN%d, memory allocation %d bytes failed\n", - __func__, __LINE__, lenOfBinary + 1); - } - rand_string(s, lenOfBinary); - pstr += sprintf(pstr, ",\"%s\"", s); - free(s); - } else if (strcasecmp(data_type[i % columnCount], "NCHAR") == 0) { - char *s = malloc(lenOfBinary + 1); - if (s == NULL) { - errorPrint("%s() LN%d, memory allocation %d bytes failed\n", - __func__, __LINE__, lenOfBinary + 1); - } - rand_string(s, lenOfBinary); - pstr += sprintf(pstr, ",\"%s\"", s); - free(s); + for (int i = 0; i < columnCount; i++) { + if (strcasecmp(data_type[i % columnCount], "TINYINT") == 0) { + pstr += sprintf(pstr, ",%d", rand_tinyint() ); + } else if (strcasecmp(data_type[i % columnCount], "SMALLINT") == 0) { + pstr += sprintf(pstr, ",%d", rand_smallint()); + } else if (strcasecmp(data_type[i % columnCount], "INT") == 0) { + pstr += sprintf(pstr, ",%d", rand_int()); + } else if (strcasecmp(data_type[i % columnCount], "BIGINT") == 0) { + pstr += sprintf(pstr, ",%" PRId64, rand_bigint()); + } else if (strcasecmp(data_type[i % columnCount], "TIMESTAMP") == 0) { + pstr += sprintf(pstr, ",%" PRId64, rand_bigint()); + } else if (strcasecmp(data_type[i % columnCount], "FLOAT") == 0) { + pstr += sprintf(pstr, ",%10.4f", rand_float()); + } else if (strcasecmp(data_type[i % columnCount], "DOUBLE") == 0) { + double t = rand_double(); + pstr += sprintf(pstr, ",%20.8f", t); + } else if (strcasecmp(data_type[i % columnCount], "BOOL") == 0) { + bool b = rand_bool() & 1; + pstr += sprintf(pstr, ",%s", b ? "true" : "false"); + } else if (strcasecmp(data_type[i % columnCount], "BINARY") == 0) { + char *s = malloc(lenOfBinary + 1); + if (s == NULL) { + errorPrint("%s() LN%d, memory allocation %d bytes failed\n", + __func__, __LINE__, lenOfBinary + 1); + exit(-1); + } + rand_string(s, lenOfBinary); + pstr += sprintf(pstr, ",\"%s\"", s); + free(s); + } else if (strcasecmp(data_type[i % columnCount], "NCHAR") == 0) { + char *s = malloc(lenOfBinary + 1); + if (s == NULL) { + errorPrint("%s() LN%d, memory allocation %d bytes failed\n", + __func__, __LINE__, lenOfBinary + 1); + exit(-1); + } + rand_string(s, lenOfBinary); + pstr += sprintf(pstr, ",\"%s\"", s); + free(s); + } + + if (strlen(recBuf) > MAX_DATA_SIZE) { + perror("column length too long, abort"); + exit(-1); + } } - if (strlen(recBuf) > MAX_DATA_SIZE) { - perror("column length too long, abort"); - exit(-1); - } - } + pstr += sprintf(pstr, ")"); - pstr += sprintf(pstr, ")"); + verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf); - verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf); - - return (int32_t)strlen(recBuf); + return (int32_t)strlen(recBuf); } static int prepareSampleDataForSTable(SSuperTable *superTblInfo) { @@ -5210,63 +5225,69 @@ static int32_t generateStbDataTail( int64_t remainderBufLen, int64_t insertRows, uint64_t recordFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) { - uint64_t len = 0; + uint64_t len = 0; - char *pstr = buffer; + char *pstr = buffer; - bool tsRand; - if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { - tsRand = true; - } else { - tsRand = false; - } - verbosePrint("%s() LN%d batch=%u buflen=%"PRId64"\n", - __func__, __LINE__, batch, remainderBufLen); - - int32_t k; - for (k = 0; k < batch;) { - char data[MAX_DATA_SIZE]; - memset(data, 0, MAX_DATA_SIZE); - - int64_t lenOfRow = 0; - - if (tsRand) { - lenOfRow = generateStbRowData(superTblInfo, data, - startTime + getTSRandTail( - superTblInfo->timeStampStep, k, - superTblInfo->disorderRatio, - superTblInfo->disorderRange) - ); + bool tsRand; + if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { + tsRand = true; } else { - lenOfRow = getRowDataFromSample( - data, - (remainderBufLen < MAX_DATA_SIZE)?remainderBufLen:MAX_DATA_SIZE, - startTime + superTblInfo->timeStampStep * k, - superTblInfo, - pSamplePos); + tsRand = false; + } + verbosePrint("%s() LN%d batch=%u buflen=%"PRId64"\n", + __func__, __LINE__, batch, remainderBufLen); + + int32_t k; + for (k = 0; k < batch;) { + char data[MAX_DATA_SIZE]; + memset(data, 0, MAX_DATA_SIZE); + + int64_t lenOfRow = 0; + + if (tsRand) { + if (superTblInfo->disorderRatio > 0) { + lenOfRow = generateStbRowData(superTblInfo, data, + startTime + getTSRandTail( + superTblInfo->timeStampStep, k, + superTblInfo->disorderRatio, + superTblInfo->disorderRange) + ); + } else { + lenOfRow = generateStbRowData(superTblInfo, data, + startTime + superTblInfo->timeStampStep * k + ); + } + } else { + lenOfRow = getRowDataFromSample( + data, + (remainderBufLen < MAX_DATA_SIZE)?remainderBufLen:MAX_DATA_SIZE, + startTime + superTblInfo->timeStampStep * k, + superTblInfo, + pSamplePos); + } + + if ((lenOfRow + 1) > remainderBufLen) { + break; + } + + pstr += snprintf(pstr , lenOfRow + 1, "%s", data); + k++; + len += lenOfRow; + remainderBufLen -= lenOfRow; + + verbosePrint("%s() LN%d len=%"PRIu64" k=%u \nbuffer=%s\n", + __func__, __LINE__, len, k, buffer); + + recordFrom ++; + + if (recordFrom >= insertRows) { + break; + } } - if ((lenOfRow + 1) > remainderBufLen) { - break; - } - - pstr += snprintf(pstr , lenOfRow + 1, "%s", data); - k++; - len += lenOfRow; - remainderBufLen -= lenOfRow; - - verbosePrint("%s() LN%d len=%"PRIu64" k=%u \nbuffer=%s\n", - __func__, __LINE__, len, k, buffer); - - recordFrom ++; - - if (recordFrom >= insertRows) { - break; - } - } - - *dataLen = len; - return k; + *dataLen = len; + return k; } @@ -5363,52 +5384,52 @@ static int32_t generateStbInterlaceData( int64_t startTime, uint64_t *pRemainderBufLen) { - assert(buffer); - char *pstr = buffer; + assert(buffer); + char *pstr = buffer; - int headLen = generateStbSQLHead( - superTblInfo, - tableName, tableSeq, pThreadInfo->db_name, - pstr, *pRemainderBufLen); + int headLen = generateStbSQLHead( + superTblInfo, + tableName, tableSeq, pThreadInfo->db_name, + pstr, *pRemainderBufLen); - if (headLen <= 0) { - return 0; - } - // generate data buffer - verbosePrint("[%d] %s() LN%d i=%"PRIu64" buffer:\n%s\n", + if (headLen <= 0) { + return 0; + } + // generate data buffer + verbosePrint("[%d] %s() LN%d i=%"PRIu64" buffer:\n%s\n", pThreadInfo->threadID, __func__, __LINE__, i, buffer); - pstr += headLen; - *pRemainderBufLen -= headLen; + pstr += headLen; + *pRemainderBufLen -= headLen; - int64_t dataLen = 0; + int64_t dataLen = 0; - verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%u batchPerTbl = %u\n", + verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%u batchPerTbl = %u\n", pThreadInfo->threadID, __func__, __LINE__, i, batchPerTblTimes, batchPerTbl); - if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { - startTime = taosGetTimestamp(pThreadInfo->time_precision); - } + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + startTime = taosGetTimestamp(pThreadInfo->time_precision); + } - int32_t k = generateStbDataTail( + int32_t k = generateStbDataTail( superTblInfo, batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, startTime, &(pThreadInfo->samplePos), &dataLen); - if (k == batchPerTbl) { - pstr += dataLen; - *pRemainderBufLen -= dataLen; - } else { - debugPrint("%s() LN%d, generated data tail: %u, not equal batch per table: %u\n", - __func__, __LINE__, k, batchPerTbl); - pstr -= headLen; - pstr[0] = '\0'; - k = 0; - } + if (k == batchPerTbl) { + pstr += dataLen; + *pRemainderBufLen -= dataLen; + } else { + debugPrint("%s() LN%d, generated data tail: %u, not equal batch per table: %u\n", + __func__, __LINE__, k, batchPerTbl); + pstr -= headLen; + pstr[0] = '\0'; + k = 0; + } - return k; + return k; } static int64_t generateInterlaceDataWithoutStb( @@ -5856,7 +5877,7 @@ static int32_t prepareStbStmtInterlace( stbInfo, stmt, tableName, - g_args.num_of_RPR, + batch, insertRows, 0, startTime, pSamplePos); } @@ -5890,29 +5911,29 @@ static int32_t generateStbProgressiveData( uint64_t recordFrom, int64_t startTime, int64_t *pSamplePos, int64_t *pRemainderBufLen) { - assert(buffer != NULL); - char *pstr = buffer; + assert(buffer != NULL); + char *pstr = buffer; - memset(buffer, 0, *pRemainderBufLen); + memset(buffer, 0, *pRemainderBufLen); - int64_t headLen = generateStbSQLHead( - superTblInfo, - tableName, tableSeq, dbName, - buffer, *pRemainderBufLen); + int64_t headLen = generateStbSQLHead( + superTblInfo, + tableName, tableSeq, dbName, + buffer, *pRemainderBufLen); - if (headLen <= 0) { - return 0; - } - pstr += headLen; - *pRemainderBufLen -= headLen; + if (headLen <= 0) { + return 0; + } + pstr += headLen; + *pRemainderBufLen -= headLen; - int64_t dataLen; + int64_t dataLen; - return generateStbDataTail(superTblInfo, - g_args.num_of_RPR, pstr, *pRemainderBufLen, - insertRows, recordFrom, - startTime, - pSamplePos, &dataLen); + return generateStbDataTail(superTblInfo, + g_args.num_of_RPR, pstr, *pRemainderBufLen, + insertRows, recordFrom, + startTime, + pSamplePos, &dataLen); } static int32_t generateProgressiveDataWithoutStb( @@ -5949,291 +5970,293 @@ static int32_t generateProgressiveDataWithoutStb( static void printStatPerThread(threadInfo *pThreadInfo) { fprintf(stderr, "====thread[%d] completed total inserted rows: %"PRIu64 ", total affected rows: %"PRIu64". %.2f records/second====\n", - pThreadInfo->threadID, - pThreadInfo->totalInsertRows, - pThreadInfo->totalAffectedRows, - (pThreadInfo->totalDelay)?(double)((pThreadInfo->totalAffectedRows / (pThreadInfo->totalDelay)/1000.0)): FLT_MAX); + pThreadInfo->threadID, + pThreadInfo->totalInsertRows, + pThreadInfo->totalAffectedRows, + (pThreadInfo->totalDelay/1000.0)? + (double)(pThreadInfo->totalAffectedRows/(pThreadInfo->totalDelay/1000.0)): + FLT_MAX); } // sync write interlace data static void* syncWriteInterlace(threadInfo *pThreadInfo) { - debugPrint("[%d] %s() LN%d: ### interlace write\n", - pThreadInfo->threadID, __func__, __LINE__); - - int64_t insertRows; - uint32_t interlaceRows; - uint64_t maxSqlLen; - int64_t nTimeStampStep; - uint64_t insert_interval; - - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - - if (superTblInfo) { - insertRows = superTblInfo->insertRows; - - if ((superTblInfo->interlaceRows == 0) - && (g_args.interlace_rows > 0)) { - interlaceRows = g_args.interlace_rows; - } else { - interlaceRows = superTblInfo->interlaceRows; - } - maxSqlLen = superTblInfo->maxSqlLen; - nTimeStampStep = superTblInfo->timeStampStep; - insert_interval = superTblInfo->insertInterval; - } else { - insertRows = g_args.num_of_DPT; - interlaceRows = g_args.interlace_rows; - maxSqlLen = g_args.max_sql_len; - nTimeStampStep = DEFAULT_TIMESTAMP_STEP; - insert_interval = g_args.insert_interval; - } - - debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", - pThreadInfo->threadID, __func__, __LINE__, - pThreadInfo->start_table_from, - pThreadInfo->ntables, insertRows); - - if (interlaceRows > insertRows) - interlaceRows = insertRows; - - if (interlaceRows > g_args.num_of_RPR) - interlaceRows = g_args.num_of_RPR; - - uint32_t batchPerTbl = interlaceRows; - uint32_t batchPerTblTimes; - - if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) { - batchPerTblTimes = - g_args.num_of_RPR / interlaceRows; - } else { - batchPerTblTimes = 1; - } - - pThreadInfo->buffer = calloc(maxSqlLen, 1); - if (NULL == pThreadInfo->buffer) { - errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n", - __func__, __LINE__, maxSqlLen, strerror(errno)); - return NULL; - } - - pThreadInfo->totalInsertRows = 0; - pThreadInfo->totalAffectedRows = 0; - - uint64_t st = 0; - uint64_t et = UINT64_MAX; - - uint64_t lastPrintTime = taosGetTimestampMs(); - uint64_t startTs = taosGetTimestampMs(); - uint64_t endTs; - - uint64_t tableSeq = pThreadInfo->start_table_from; - int64_t startTime = pThreadInfo->start_time; - - uint64_t generatedRecPerTbl = 0; - bool flagSleep = true; - uint64_t sleepTimeTotal = 0; - - while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { - if ((flagSleep) && (insert_interval)) { - st = taosGetTimestampMs(); - flagSleep = false; - } - // generate data - memset(pThreadInfo->buffer, 0, maxSqlLen); - uint64_t remainderBufLen = maxSqlLen; - - char *pstr = pThreadInfo->buffer; - - int len = snprintf(pstr, - strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO); - pstr += len; - remainderBufLen -= len; - - uint32_t recOfBatch = 0; - - for (uint64_t i = 0; i < batchPerTblTimes; i ++) { - char tableName[TSDB_TABLE_NAME_LEN]; - - getTableName(tableName, pThreadInfo, tableSeq); - if (0 == strlen(tableName)) { - errorPrint("[%d] %s() LN%d, getTableName return null\n", + debugPrint("[%d] %s() LN%d: ### interlace write\n", pThreadInfo->threadID, __func__, __LINE__); - free(pThreadInfo->buffer); - return NULL; - } - uint64_t oldRemainderLen = remainderBufLen; + int64_t insertRows; + uint32_t interlaceRows; + uint64_t maxSqlLen; + int64_t nTimeStampStep; + uint64_t insert_interval; - int32_t generated; - if (superTblInfo) { - if (superTblInfo->iface == STMT_IFACE) { -#if STMT_IFACE_ENABLED == 1 - generated = prepareStbStmtInterlace( - superTblInfo, - pThreadInfo->stmt, - tableName, - batchPerTbl, - insertRows, i, - startTime, - &(pThreadInfo->samplePos)); -#else - generated = -1; -#endif - } else { - generated = generateStbInterlaceData( - superTblInfo, - tableName, batchPerTbl, i, - batchPerTblTimes, - tableSeq, - pThreadInfo, pstr, - insertRows, - startTime, - &remainderBufLen); - } - } else { - if (g_args.iface == STMT_IFACE) { - debugPrint("[%d] %s() LN%d, tableName:%s, batch:%d startTime:%"PRId64"\n", - pThreadInfo->threadID, - __func__, __LINE__, - tableName, batchPerTbl, startTime); -#if STMT_IFACE_ENABLED == 1 - generated = prepareStmtWithoutStb( - pThreadInfo->stmt, tableName, - batchPerTbl, - insertRows, i, - startTime); -#else - generated = -1; -#endif - } else { - generated = generateInterlaceDataWithoutStb( - tableName, batchPerTbl, - tableSeq, - pThreadInfo->db_name, pstr, - insertRows, - startTime, - &remainderBufLen); - } - } + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - debugPrint("[%d] %s() LN%d, generated records is %d\n", - pThreadInfo->threadID, __func__, __LINE__, generated); - if (generated < 0) { - errorPrint("[%d] %s() LN%d, generated records is %d\n", - pThreadInfo->threadID, __func__, __LINE__, generated); - goto free_of_interlace; - } else if (generated == 0) { - break; - } + if (superTblInfo) { + insertRows = superTblInfo->insertRows; - tableSeq ++; - recOfBatch += batchPerTbl; - - pstr += (oldRemainderLen - remainderBufLen); - pThreadInfo->totalInsertRows += batchPerTbl; - - verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n", - pThreadInfo->threadID, __func__, __LINE__, - batchPerTbl, recOfBatch); - - if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { - // turn to first table - tableSeq = pThreadInfo->start_table_from; - generatedRecPerTbl += batchPerTbl; - - startTime = pThreadInfo->start_time - + generatedRecPerTbl * nTimeStampStep; - - flagSleep = true; - if (generatedRecPerTbl >= insertRows) - break; - - int64_t remainRows = insertRows - generatedRecPerTbl; - if ((remainRows > 0) && (batchPerTbl > remainRows)) - batchPerTbl = remainRows; - - if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR) - break; - } - - verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n", - pThreadInfo->threadID, __func__, __LINE__, - generatedRecPerTbl, insertRows); - - if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl) - break; - } - - verbosePrint("[%d] %s() LN%d recOfBatch=%d totalInsertRows=%"PRIu64"\n", - pThreadInfo->threadID, __func__, __LINE__, recOfBatch, - pThreadInfo->totalInsertRows); - verbosePrint("[%d] %s() LN%d, buffer=%s\n", - pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer); - - startTs = taosGetTimestampMs(); - - if (recOfBatch == 0) { - errorPrint("[%d] %s() LN%d Failed to insert records of batch %d\n", - pThreadInfo->threadID, __func__, __LINE__, - batchPerTbl); - if (batchPerTbl > 0) { - errorPrint("\tIf the batch is %d, the length of the SQL to insert a row must be less then %"PRId64"\n", - batchPerTbl, maxSqlLen / batchPerTbl); + if ((superTblInfo->interlaceRows == 0) + && (g_args.interlace_rows > 0)) { + interlaceRows = g_args.interlace_rows; + } else { + interlaceRows = superTblInfo->interlaceRows; } - errorPrint("\tPlease check if the buffer length(%"PRId64") or batch(%d) is set with proper value!\n", - maxSqlLen, batchPerTbl); - goto free_of_interlace; - } - int64_t affectedRows = execInsert(pThreadInfo, recOfBatch); - - endTs = taosGetTimestampMs(); - uint64_t delay = endTs - startTs; - performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n", - __func__, __LINE__, delay); - verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n", - pThreadInfo->threadID, - __func__, __LINE__, affectedRows); - - if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay; - if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay; - pThreadInfo->cntDelay++; - pThreadInfo->totalDelay += delay; - - if (recOfBatch != affectedRows) { - errorPrint("[%d] %s() LN%d execInsert insert %d, affected rows: %"PRId64"\n%s\n", - pThreadInfo->threadID, __func__, __LINE__, - recOfBatch, affectedRows, pThreadInfo->buffer); - goto free_of_interlace; + maxSqlLen = superTblInfo->maxSqlLen; + nTimeStampStep = superTblInfo->timeStampStep; + insert_interval = superTblInfo->insertInterval; + } else { + insertRows = g_args.num_of_DPT; + interlaceRows = g_args.interlace_rows; + maxSqlLen = g_args.max_sql_len; + nTimeStampStep = DEFAULT_TIMESTAMP_STEP; + insert_interval = g_args.insert_interval; } - pThreadInfo->totalAffectedRows += affectedRows; + debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", + pThreadInfo->threadID, __func__, __LINE__, + pThreadInfo->start_table_from, + pThreadInfo->ntables, insertRows); - int64_t currentPrintTime = taosGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 30*1000) { - printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n", + if (interlaceRows > insertRows) + interlaceRows = insertRows; + + if (interlaceRows > g_args.num_of_RPR) + interlaceRows = g_args.num_of_RPR; + + uint32_t batchPerTbl = interlaceRows; + uint32_t batchPerTblTimes; + + if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) { + batchPerTblTimes = + g_args.num_of_RPR / interlaceRows; + } else { + batchPerTblTimes = 1; + } + + pThreadInfo->buffer = calloc(maxSqlLen, 1); + if (NULL == pThreadInfo->buffer) { + errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n", + __func__, __LINE__, maxSqlLen, strerror(errno)); + return NULL; + } + + pThreadInfo->totalInsertRows = 0; + pThreadInfo->totalAffectedRows = 0; + + uint64_t st = 0; + uint64_t et = UINT64_MAX; + + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); + uint64_t endTs; + + uint64_t tableSeq = pThreadInfo->start_table_from; + int64_t startTime = pThreadInfo->start_time; + + uint64_t generatedRecPerTbl = 0; + bool flagSleep = true; + uint64_t sleepTimeTotal = 0; + + while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { + if ((flagSleep) && (insert_interval)) { + st = taosGetTimestampMs(); + flagSleep = false; + } + // generate data + memset(pThreadInfo->buffer, 0, maxSqlLen); + uint64_t remainderBufLen = maxSqlLen; + + char *pstr = pThreadInfo->buffer; + + int len = snprintf(pstr, + strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO); + pstr += len; + remainderBufLen -= len; + + uint32_t recOfBatch = 0; + + for (uint64_t i = 0; i < batchPerTblTimes; i ++) { + char tableName[TSDB_TABLE_NAME_LEN]; + + getTableName(tableName, pThreadInfo, tableSeq); + if (0 == strlen(tableName)) { + errorPrint("[%d] %s() LN%d, getTableName return null\n", + pThreadInfo->threadID, __func__, __LINE__); + free(pThreadInfo->buffer); + return NULL; + } + + uint64_t oldRemainderLen = remainderBufLen; + + int32_t generated; + if (superTblInfo) { + if (superTblInfo->iface == STMT_IFACE) { +#if STMT_IFACE_ENABLED == 1 + generated = prepareStbStmtInterlace( + superTblInfo, + pThreadInfo->stmt, + tableName, + batchPerTbl, + insertRows, i, + startTime, + &(pThreadInfo->samplePos)); +#else + generated = -1; +#endif + } else { + generated = generateStbInterlaceData( + superTblInfo, + tableName, batchPerTbl, i, + batchPerTblTimes, + tableSeq, + pThreadInfo, pstr, + insertRows, + startTime, + &remainderBufLen); + } + } else { + if (g_args.iface == STMT_IFACE) { + debugPrint("[%d] %s() LN%d, tableName:%s, batch:%d startTime:%"PRId64"\n", + pThreadInfo->threadID, + __func__, __LINE__, + tableName, batchPerTbl, startTime); +#if STMT_IFACE_ENABLED == 1 + generated = prepareStmtWithoutStb( + pThreadInfo->stmt, tableName, + batchPerTbl, + insertRows, i, + startTime); +#else + generated = -1; +#endif + } else { + generated = generateInterlaceDataWithoutStb( + tableName, batchPerTbl, + tableSeq, + pThreadInfo->db_name, pstr, + insertRows, + startTime, + &remainderBufLen); + } + } + + debugPrint("[%d] %s() LN%d, generated records is %d\n", + pThreadInfo->threadID, __func__, __LINE__, generated); + if (generated < 0) { + errorPrint("[%d] %s() LN%d, generated records is %d\n", + pThreadInfo->threadID, __func__, __LINE__, generated); + goto free_of_interlace; + } else if (generated == 0) { + break; + } + + tableSeq ++; + recOfBatch += batchPerTbl; + + pstr += (oldRemainderLen - remainderBufLen); + pThreadInfo->totalInsertRows += batchPerTbl; + + verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n", + pThreadInfo->threadID, __func__, __LINE__, + batchPerTbl, recOfBatch); + + if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { + // turn to first table + tableSeq = pThreadInfo->start_table_from; + generatedRecPerTbl += batchPerTbl; + + startTime = pThreadInfo->start_time + + generatedRecPerTbl * nTimeStampStep; + + flagSleep = true; + if (generatedRecPerTbl >= insertRows) + break; + + int64_t remainRows = insertRows - generatedRecPerTbl; + if ((remainRows > 0) && (batchPerTbl > remainRows)) + batchPerTbl = remainRows; + + if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR) + break; + } + + verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n", + pThreadInfo->threadID, __func__, __LINE__, + generatedRecPerTbl, insertRows); + + if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl) + break; + } + + verbosePrint("[%d] %s() LN%d recOfBatch=%d totalInsertRows=%"PRIu64"\n", + pThreadInfo->threadID, __func__, __LINE__, recOfBatch, + pThreadInfo->totalInsertRows); + verbosePrint("[%d] %s() LN%d, buffer=%s\n", + pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer); + + startTs = taosGetTimestampMs(); + + if (recOfBatch == 0) { + errorPrint("[%d] %s() LN%d Failed to insert records of batch %d\n", + pThreadInfo->threadID, __func__, __LINE__, + batchPerTbl); + if (batchPerTbl > 0) { + errorPrint("\tIf the batch is %d, the length of the SQL to insert a row must be less then %"PRId64"\n", + batchPerTbl, maxSqlLen / batchPerTbl); + } + errorPrint("\tPlease check if the buffer length(%"PRId64") or batch(%d) is set with proper value!\n", + maxSqlLen, batchPerTbl); + goto free_of_interlace; + } + int64_t affectedRows = execInsert(pThreadInfo, recOfBatch); + + endTs = taosGetTimestampMs(); + uint64_t delay = endTs - startTs; + performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n", + __func__, __LINE__, delay); + verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n", + pThreadInfo->threadID, + __func__, __LINE__, affectedRows); + + if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay; + if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay; + pThreadInfo->cntDelay++; + pThreadInfo->totalDelay += delay; + + if (recOfBatch != affectedRows) { + errorPrint("[%d] %s() LN%d execInsert insert %d, affected rows: %"PRId64"\n%s\n", + pThreadInfo->threadID, __func__, __LINE__, + recOfBatch, affectedRows, pThreadInfo->buffer); + goto free_of_interlace; + } + + pThreadInfo->totalAffectedRows += affectedRows; + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n", pThreadInfo->threadID, pThreadInfo->totalInsertRows, pThreadInfo->totalAffectedRows); - lastPrintTime = currentPrintTime; - } + lastPrintTime = currentPrintTime; + } - if ((insert_interval) && flagSleep) { - et = taosGetTimestampMs(); + if ((insert_interval) && flagSleep) { + et = taosGetTimestampMs(); - if (insert_interval > (et - st) ) { - uint64_t sleepTime = insert_interval - (et -st); - performancePrint("%s() LN%d sleep: %"PRId64" ms for insert interval\n", - __func__, __LINE__, sleepTime); - taosMsleep(sleepTime); // ms - sleepTimeTotal += insert_interval; - } + if (insert_interval > (et - st) ) { + uint64_t sleepTime = insert_interval - (et -st); + performancePrint("%s() LN%d sleep: %"PRId64" ms for insert interval\n", + __func__, __LINE__, sleepTime); + taosMsleep(sleepTime); // ms + sleepTimeTotal += insert_interval; + } + } } - } free_of_interlace: - tmfree(pThreadInfo->buffer); - printStatPerThread(pThreadInfo); - return NULL; + tmfree(pThreadInfo->buffer); + printStatPerThread(pThreadInfo); + return NULL; } // sync insertion progressive data @@ -6400,29 +6423,29 @@ free_of_progressive: static void* syncWrite(void *sarg) { - threadInfo *pThreadInfo = (threadInfo *)sarg; - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + threadInfo *pThreadInfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - uint32_t interlaceRows; + uint32_t interlaceRows; - if (superTblInfo) { - if ((superTblInfo->interlaceRows == 0) - && (g_args.interlace_rows > 0)) { - interlaceRows = g_args.interlace_rows; + if (superTblInfo) { + if ((superTblInfo->interlaceRows == 0) + && (g_args.interlace_rows > 0)) { + interlaceRows = g_args.interlace_rows; + } else { + interlaceRows = superTblInfo->interlaceRows; + } } else { - interlaceRows = superTblInfo->interlaceRows; + interlaceRows = g_args.interlace_rows; } - } else { - interlaceRows = g_args.interlace_rows; - } - if (interlaceRows > 0) { - // interlace mode - return syncWriteInterlace(pThreadInfo); - } else { - // progressive mode - return syncWriteProgressive(pThreadInfo); - } + if (interlaceRows > 0) { + // interlace mode + return syncWriteInterlace(pThreadInfo); + } else { + // progressive mode + return syncWriteProgressive(pThreadInfo); + } } static void callBack(void *param, TAOS_RES *res, int code) { @@ -6533,343 +6556,353 @@ static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in * static void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSuperTable* superTblInfo) { - int32_t timePrec = TSDB_TIME_PRECISION_MILLI; - if (0 != precision[0]) { - if (0 == strncasecmp(precision, "ms", 2)) { - timePrec = TSDB_TIME_PRECISION_MILLI; - } else if (0 == strncasecmp(precision, "us", 2)) { - timePrec = TSDB_TIME_PRECISION_MICRO; - } else if (0 == strncasecmp(precision, "ns", 2)) { - timePrec = TSDB_TIME_PRECISION_NANO; - } else { - errorPrint("Not support precision: %s\n", precision); - exit(-1); - } - } - - int64_t start_time; - if (superTblInfo) { - if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { - start_time = taosGetTimestamp(timePrec); - } else { - if (TSDB_CODE_SUCCESS != taosParseTime( - superTblInfo->startTimestamp, - &start_time, - strlen(superTblInfo->startTimestamp), - timePrec, 0)) { - ERROR_EXIT("failed to parse time!\n"); - } - } - } else { - start_time = 1500000000000; - } - - int64_t start = taosGetTimestampMs(); - - // read sample data from file first - if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource, - "sample", strlen("sample")))) { - if (0 != prepareSampleDataForSTable(superTblInfo)) { - errorPrint("%s() LN%d, prepare sample data for stable failed!\n", - __func__, __LINE__); - exit(-1); - } - } - - TAOS* taos0 = taos_connect( - g_Dbs.host, g_Dbs.user, - g_Dbs.password, db_name, g_Dbs.port); - if (NULL == taos0) { - errorPrint("%s() LN%d, connect to server fail , reason: %s\n", - __func__, __LINE__, taos_errstr(NULL)); - exit(-1); - } - - int64_t ntables = 0; - uint64_t tableFrom; - - if (superTblInfo) { - int64_t limit; - uint64_t offset; - - if ((NULL != g_args.sqlFile) && (superTblInfo->childTblExists == TBL_NO_EXISTS) && - ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit >= 0))) { - printf("WARNING: offset and limit will not be used since the child tables not exists!\n"); - } - - if (superTblInfo->childTblExists == TBL_ALREADY_EXISTS) { - if ((superTblInfo->childTblLimit < 0) - || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit) - > (superTblInfo->childTblCount))) { - superTblInfo->childTblLimit = - superTblInfo->childTblCount - superTblInfo->childTblOffset; - } - - offset = superTblInfo->childTblOffset; - limit = superTblInfo->childTblLimit; - } else { - limit = superTblInfo->childTblCount; - offset = 0; - } - - ntables = limit; - tableFrom = offset; - - if ((superTblInfo->childTblExists != TBL_NO_EXISTS) - && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit ) - > superTblInfo->childTblCount)) { - printf("WARNING: specified offset + limit > child table count!\n"); - prompt(); - } - - if ((superTblInfo->childTblExists != TBL_NO_EXISTS) - && (0 == superTblInfo->childTblLimit)) { - printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n"); - prompt(); - } - - superTblInfo->childTblName = (char*)calloc(1, - limit * TSDB_TABLE_NAME_LEN); - if (superTblInfo->childTblName == NULL) { - errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__); - taos_close(taos0); - exit(-1); - } - - int64_t childTblCount; - getChildNameOfSuperTableWithLimitAndOffset( - taos0, - db_name, superTblInfo->sTblName, - &superTblInfo->childTblName, &childTblCount, - limit, - offset); - } else { - ntables = g_args.num_of_tables; - tableFrom = 0; - } - - taos_close(taos0); - - int64_t a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; - } - - int64_t b = 0; - if (threads != 0) { - b = ntables % threads; - } - - if ((superTblInfo) - && (superTblInfo->iface == REST_IFACE)) { - if (convertHostToServAddr( - g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) { - exit(-1); - } - } - - pthread_t *pids = calloc(1, threads * sizeof(pthread_t)); - assert(pids != NULL); - - threadInfo *infos = calloc(1, threads * sizeof(threadInfo)); - assert(infos != NULL); - - memset(pids, 0, threads * sizeof(pthread_t)); - memset(infos, 0, threads * sizeof(threadInfo)); - - for (int i = 0; i < threads; i++) { - threadInfo *pThreadInfo = infos + i; - pThreadInfo->threadID = i; - tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN); - pThreadInfo->time_precision = timePrec; - pThreadInfo->superTblInfo = superTblInfo; - - pThreadInfo->start_time = start_time; - pThreadInfo->minDelay = UINT64_MAX; - - if ((NULL == superTblInfo) || - (superTblInfo->iface != REST_IFACE)) { - //t_info->taos = taos; - pThreadInfo->taos = taos_connect( - g_Dbs.host, g_Dbs.user, - g_Dbs.password, db_name, g_Dbs.port); - if (NULL == pThreadInfo->taos) { - errorPrint( - "%s() LN%d, connect to server fail from insert sub thread, reason: %s\n", - __func__, __LINE__, - taos_errstr(NULL)); - free(infos); - exit(-1); - } - -#if STMT_IFACE_ENABLED == 1 - if ((g_args.iface == STMT_IFACE) - || ((superTblInfo) && (superTblInfo->iface == STMT_IFACE))) { - - int columnCount; - if (superTblInfo) { - columnCount = superTblInfo->columnCount; - } else { - columnCount = g_args.num_of_CPR; - } - - pThreadInfo->stmt = taos_stmt_init(pThreadInfo->taos); - if (NULL == pThreadInfo->stmt) { - errorPrint( - "%s() LN%d, failed init stmt, reason: %s\n", - __func__, __LINE__, - taos_errstr(NULL)); - free(pids); - free(infos); - exit(-1); - } - - char buffer[3000]; - char *pstr = buffer; - - if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { - pstr += sprintf(pstr, "INSERT INTO ? USING %s TAGS(?", - superTblInfo->sTblName); - for (int tag = 0; tag < (superTblInfo->tagCount - 1); tag ++ ) { - pstr += sprintf(pstr, ",?"); - } - pstr += sprintf(pstr, ") VALUES(?"); + int32_t timePrec = TSDB_TIME_PRECISION_MILLI; + if (0 != precision[0]) { + if (0 == strncasecmp(precision, "ms", 2)) { + timePrec = TSDB_TIME_PRECISION_MILLI; + } else if (0 == strncasecmp(precision, "us", 2)) { + timePrec = TSDB_TIME_PRECISION_MICRO; +#if NANO_SECOND_ENABLED == 1 + } else if (0 == strncasecmp(precision, "ns", 2)) { + timePrec = TSDB_TIME_PRECISION_NANO; +#endif } else { - pstr += sprintf(pstr, "INSERT INTO ? VALUES(?"); - } - - for (int col = 0; col < columnCount; col ++) { - pstr += sprintf(pstr, ",?"); - } - pstr += sprintf(pstr, ")"); - - debugPrint("%s() LN%d, buffer: %s", __func__, __LINE__, buffer); - int ret = taos_stmt_prepare(pThreadInfo->stmt, buffer, 0); - if (ret != 0){ - errorPrint("failed to execute taos_stmt_prepare. return 0x%x. reason: %s\n", - ret, taos_errstr(NULL)); - free(pids); - free(infos); + errorPrint("Not support precision: %s\n", precision); exit(-1); } - } -#endif + } + + int64_t start_time; + if (superTblInfo) { + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + start_time = taosGetTimestamp(timePrec); + } else { + if (TSDB_CODE_SUCCESS != taosParseTime( + superTblInfo->startTimestamp, + &start_time, + strlen(superTblInfo->startTimestamp), + timePrec, 0)) { + ERROR_EXIT("failed to parse time!\n"); + } + } } else { - pThreadInfo->taos = NULL; + start_time = 1500000000000; } -/* if ((NULL == superTblInfo) - || (0 == superTblInfo->multiThreadWriteOneTbl)) { - */ - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = iend_table_to = i < b ? tableFrom + a : tableFrom + a - 1; - tableFrom = pThreadInfo->end_table_to + 1; -/* } else { - pThreadInfo->start_table_from = 0; - pThreadInfo->ntables = superTblInfo->childTblCount; - pThreadInfo->start_time = pThreadInfo->start_time + rand_int() % 10000 - rand_tinyint(); + int64_t start = taosGetTimestampMs(); + + // read sample data from file first + if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource, + "sample", strlen("sample")))) { + if (0 != prepareSampleDataForSTable(superTblInfo)) { + errorPrint("%s() LN%d, prepare sample data for stable failed!\n", + __func__, __LINE__); + exit(-1); + } } -*/ - tsem_init(&(pThreadInfo->lock_sem), 0, 0); - if (ASYNC_MODE == g_Dbs.asyncMode) { - pthread_create(pids + i, NULL, asyncWrite, pThreadInfo); + + TAOS* taos0 = taos_connect( + g_Dbs.host, g_Dbs.user, + g_Dbs.password, db_name, g_Dbs.port); + if (NULL == taos0) { + errorPrint("%s() LN%d, connect to server fail , reason: %s\n", + __func__, __LINE__, taos_errstr(NULL)); + exit(-1); + } + + int64_t ntables = 0; + uint64_t tableFrom; + + if (superTblInfo) { + int64_t limit; + uint64_t offset; + + if ((NULL != g_args.sqlFile) && (superTblInfo->childTblExists == TBL_NO_EXISTS) && + ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit >= 0))) { + printf("WARNING: offset and limit will not be used since the child tables not exists!\n"); + } + + if (superTblInfo->childTblExists == TBL_ALREADY_EXISTS) { + if ((superTblInfo->childTblLimit < 0) + || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit) + > (superTblInfo->childTblCount))) { + superTblInfo->childTblLimit = + superTblInfo->childTblCount - superTblInfo->childTblOffset; + } + + offset = superTblInfo->childTblOffset; + limit = superTblInfo->childTblLimit; + } else { + limit = superTblInfo->childTblCount; + offset = 0; + } + + ntables = limit; + tableFrom = offset; + + if ((superTblInfo->childTblExists != TBL_NO_EXISTS) + && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit ) + > superTblInfo->childTblCount)) { + printf("WARNING: specified offset + limit > child table count!\n"); + prompt(); + } + + if ((superTblInfo->childTblExists != TBL_NO_EXISTS) + && (0 == superTblInfo->childTblLimit)) { + printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n"); + prompt(); + } + + superTblInfo->childTblName = (char*)calloc(1, + limit * TSDB_TABLE_NAME_LEN); + if (superTblInfo->childTblName == NULL) { + errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__); + taos_close(taos0); + exit(-1); + } + + int64_t childTblCount; + getChildNameOfSuperTableWithLimitAndOffset( + taos0, + db_name, superTblInfo->sTblName, + &superTblInfo->childTblName, &childTblCount, + limit, + offset); } else { - pthread_create(pids + i, NULL, syncWrite, pThreadInfo); + ntables = g_args.num_of_tables; + tableFrom = 0; } - } - for (int i = 0; i < threads; i++) { - pthread_join(pids[i], NULL); - } + taos_close(taos0); - uint64_t totalDelay = 0; - uint64_t maxDelay = 0; - uint64_t minDelay = UINT64_MAX; - uint64_t cntDelay = 1; - double avgDelay = 0; + int64_t a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } - for (int i = 0; i < threads; i++) { - threadInfo *pThreadInfo = infos + i; + int64_t b = 0; + if (threads != 0) { + b = ntables % threads; + } - tsem_destroy(&(pThreadInfo->lock_sem)); + if ((superTblInfo) + && (superTblInfo->iface == REST_IFACE)) { + if (convertHostToServAddr( + g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) { + exit(-1); + } + } + + pthread_t *pids = calloc(1, threads * sizeof(pthread_t)); + assert(pids != NULL); + + threadInfo *infos = calloc(1, threads * sizeof(threadInfo)); + assert(infos != NULL); + + memset(pids, 0, threads * sizeof(pthread_t)); + memset(infos, 0, threads * sizeof(threadInfo)); + + for (int i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + pThreadInfo->threadID = i; + tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN); + pThreadInfo->time_precision = timePrec; + pThreadInfo->superTblInfo = superTblInfo; + + pThreadInfo->start_time = start_time; + pThreadInfo->minDelay = UINT64_MAX; + + if ((NULL == superTblInfo) || + (superTblInfo->iface != REST_IFACE)) { + //t_info->taos = taos; + pThreadInfo->taos = taos_connect( + g_Dbs.host, g_Dbs.user, + g_Dbs.password, db_name, g_Dbs.port); + if (NULL == pThreadInfo->taos) { + errorPrint( + "%s() LN%d, connect to server fail from insert sub thread, reason: %s\n", + __func__, __LINE__, + taos_errstr(NULL)); + free(infos); + exit(-1); + } #if STMT_IFACE_ENABLED == 1 - if (pThreadInfo->stmt) { - taos_stmt_close(pThreadInfo->stmt); - } + if ((g_args.iface == STMT_IFACE) + || ((superTblInfo) && (superTblInfo->iface == STMT_IFACE))) { + + int columnCount; + if (superTblInfo) { + columnCount = superTblInfo->columnCount; + } else { + columnCount = g_args.num_of_CPR; + } + + pThreadInfo->stmt = taos_stmt_init(pThreadInfo->taos); + if (NULL == pThreadInfo->stmt) { + errorPrint( + "%s() LN%d, failed init stmt, reason: %s\n", + __func__, __LINE__, + taos_errstr(NULL)); + free(pids); + free(infos); + exit(-1); + } + + char buffer[3000]; + char *pstr = buffer; + + if ((superTblInfo) + && (AUTO_CREATE_SUBTBL + == superTblInfo->autoCreateTable)) { + pstr += sprintf(pstr, "INSERT INTO ? USING %s TAGS(?", + superTblInfo->sTblName); + for (int tag = 0; tag < (superTblInfo->tagCount - 1); tag ++ ) { + pstr += sprintf(pstr, ",?"); + } + pstr += sprintf(pstr, ") VALUES(?"); + } else { + pstr += sprintf(pstr, "INSERT INTO ? VALUES(?"); + } + + for (int col = 0; col < columnCount; col ++) { + pstr += sprintf(pstr, ",?"); + } + pstr += sprintf(pstr, ")"); + + debugPrint("%s() LN%d, buffer: %s", __func__, __LINE__, buffer); + int ret = taos_stmt_prepare(pThreadInfo->stmt, buffer, 0); + if (ret != 0){ + errorPrint("failed to execute taos_stmt_prepare. return 0x%x. reason: %s\n", + ret, taos_errstr(NULL)); + free(pids); + free(infos); + exit(-1); + } + } #endif - tsem_destroy(&(pThreadInfo->lock_sem)); - taos_close(pThreadInfo->taos); + } else { + pThreadInfo->taos = NULL; + } + + /* if ((NULL == superTblInfo) + || (0 == superTblInfo->multiThreadWriteOneTbl)) { + */ + pThreadInfo->start_table_from = tableFrom; + pThreadInfo->ntables = iend_table_to = i < b ? tableFrom + a : tableFrom + a - 1; + tableFrom = pThreadInfo->end_table_to + 1; + /* } else { + pThreadInfo->start_table_from = 0; + pThreadInfo->ntables = superTblInfo->childTblCount; + pThreadInfo->start_time = pThreadInfo->start_time + rand_int() % 10000 - rand_tinyint(); + } + */ + tsem_init(&(pThreadInfo->lock_sem), 0, 0); + if (ASYNC_MODE == g_Dbs.asyncMode) { + pthread_create(pids + i, NULL, asyncWrite, pThreadInfo); + } else { + pthread_create(pids + i, NULL, syncWrite, pThreadInfo); + } + } + + for (int i = 0; i < threads; i++) { + pthread_join(pids[i], NULL); + } + + uint64_t totalDelay = 0; + uint64_t maxDelay = 0; + uint64_t minDelay = UINT64_MAX; + uint64_t cntDelay = 1; + double avgDelay = 0; + + for (int i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + + tsem_destroy(&(pThreadInfo->lock_sem)); + +#if STMT_IFACE_ENABLED == 1 + if (pThreadInfo->stmt) { + taos_stmt_close(pThreadInfo->stmt); + } +#endif + tsem_destroy(&(pThreadInfo->lock_sem)); + taos_close(pThreadInfo->taos); + + debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n", + __func__, __LINE__, + pThreadInfo->threadID, pThreadInfo->totalInsertRows, + pThreadInfo->totalAffectedRows); + if (superTblInfo) { + superTblInfo->totalAffectedRows += pThreadInfo->totalAffectedRows; + superTblInfo->totalInsertRows += pThreadInfo->totalInsertRows; + } else { + g_args.totalAffectedRows += pThreadInfo->totalAffectedRows; + g_args.totalInsertRows += pThreadInfo->totalInsertRows; + } + + totalDelay += pThreadInfo->totalDelay; + cntDelay += pThreadInfo->cntDelay; + if (pThreadInfo->maxDelay > maxDelay) maxDelay = pThreadInfo->maxDelay; + if (pThreadInfo->minDelay < minDelay) minDelay = pThreadInfo->minDelay; + } + cntDelay -= 1; + + if (cntDelay == 0) cntDelay = 1; + avgDelay = (double)totalDelay / cntDelay; + + int64_t end = taosGetTimestampMs(); + int64_t t = end - start; + + double tInMs = t/1000.0; - debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n", - __func__, __LINE__, - pThreadInfo->threadID, pThreadInfo->totalInsertRows, - pThreadInfo->totalAffectedRows); if (superTblInfo) { - superTblInfo->totalAffectedRows += pThreadInfo->totalAffectedRows; - superTblInfo->totalInsertRows += pThreadInfo->totalInsertRows; + fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n", + tInMs, superTblInfo->totalInsertRows, + superTblInfo->totalAffectedRows, + threads, db_name, superTblInfo->sTblName, + (tInMs)? + (double)(superTblInfo->totalInsertRows/tInMs):FLT_MAX); + + if (g_fpOfInsertResult) { + fprintf(g_fpOfInsertResult, + "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n", + tInMs, superTblInfo->totalInsertRows, + superTblInfo->totalAffectedRows, + threads, db_name, superTblInfo->sTblName, + (tInMs)? + (double)(superTblInfo->totalInsertRows/tInMs):FLT_MAX); + } } else { - g_args.totalAffectedRows += pThreadInfo->totalAffectedRows; - g_args.totalInsertRows += pThreadInfo->totalInsertRows; + fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n", + tInMs, g_args.totalInsertRows, + g_args.totalAffectedRows, + threads, db_name, + (tInMs)? + (double)(g_args.totalInsertRows/tInMs):FLT_MAX); + if (g_fpOfInsertResult) { + fprintf(g_fpOfInsertResult, + "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n", + tInMs, g_args.totalInsertRows, + g_args.totalAffectedRows, + threads, db_name, + (tInMs)? + (double)(g_args.totalInsertRows/tInMs):FLT_MAX); + } } - totalDelay += pThreadInfo->totalDelay; - cntDelay += pThreadInfo->cntDelay; - if (pThreadInfo->maxDelay > maxDelay) maxDelay = pThreadInfo->maxDelay; - if (pThreadInfo->minDelay < minDelay) minDelay = pThreadInfo->minDelay; - } - cntDelay -= 1; - - if (cntDelay == 0) cntDelay = 1; - avgDelay = (double)totalDelay / cntDelay; - - int64_t end = taosGetTimestampMs(); - int64_t t = end - start; - - if (superTblInfo) { - fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n", - t / 1000.0, superTblInfo->totalInsertRows, - superTblInfo->totalAffectedRows, - threads, db_name, superTblInfo->sTblName, - (double)superTblInfo->totalInsertRows / (t / 1000.0)); - + fprintf(stderr, "insert delay, avg: %10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n", + avgDelay, maxDelay, minDelay); if (g_fpOfInsertResult) { - fprintf(g_fpOfInsertResult, - "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n", - t / 1000.0, superTblInfo->totalInsertRows, - superTblInfo->totalAffectedRows, - threads, db_name, superTblInfo->sTblName, - (double)superTblInfo->totalInsertRows / (t / 1000.0)); + fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n", + avgDelay, maxDelay, minDelay); } - } else { - fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n", - t / 1000.0, g_args.totalInsertRows, - g_args.totalAffectedRows, - threads, db_name, - (double)g_args.totalInsertRows / (t / 1000.0)); - if (g_fpOfInsertResult) { - fprintf(g_fpOfInsertResult, - "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n", - t * 1000.0, g_args.totalInsertRows, - g_args.totalAffectedRows, - threads, db_name, - (double)g_args.totalInsertRows / (t / 1000.0)); - } - } - fprintf(stderr, "insert delay, avg: %10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n", - avgDelay, maxDelay, minDelay); - if (g_fpOfInsertResult) { - fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n", - avgDelay, maxDelay, minDelay); - } + //taos_close(taos); - //taos_close(taos); - - free(pids); - free(infos); + free(pids); + free(infos); } static void *readTable(void *sarg) { @@ -7027,98 +7060,98 @@ static void prompt() static int insertTestProcess() { - setupForAnsiEscape(); - int ret = printfInsertMeta(); - resetAfterAnsiEscape(); + setupForAnsiEscape(); + int ret = printfInsertMeta(); + resetAfterAnsiEscape(); - if (ret == -1) - exit(EXIT_FAILURE); + if (ret == -1) + exit(EXIT_FAILURE); - debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile); - g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a"); - if (NULL == g_fpOfInsertResult) { - errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile); - return -1; - } - - if (g_fpOfInsertResult) - printfInsertMetaToFile(g_fpOfInsertResult); - - prompt(); - - init_rand_data(); - - // create database and super tables - if(createDatabasesAndStables() != 0) { - if (g_fpOfInsertResult) - fclose(g_fpOfInsertResult); - return -1; - } - - // pretreatement - if (prepareSampleData() != 0) { - if (g_fpOfInsertResult) - fclose(g_fpOfInsertResult); - return -1; - } - - double start; - double end; - - // create child tables - start = taosGetTimestampMs(); - createChildTables(); - end = taosGetTimestampMs(); - - if (g_totalChildTables > 0) { - fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n", - (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl); - if (g_fpOfInsertResult) { - fprintf(g_fpOfInsertResult, - "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n", - (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl); + debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile); + g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a"); + if (NULL == g_fpOfInsertResult) { + errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile); + return -1; } - } - // create sub threads for inserting data - //start = taosGetTimestampMs(); - for (int i = 0; i < g_Dbs.dbCount; i++) { - if (g_Dbs.use_metric) { - if (g_Dbs.db[i].superTblCount > 0) { - for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { + if (g_fpOfInsertResult) + printfInsertMetaToFile(g_fpOfInsertResult); - SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j]; + prompt(); - if (superTblInfo && (superTblInfo->insertRows > 0)) { - startMultiThreadInsertData( - g_Dbs.threadCount, - g_Dbs.db[i].dbName, - g_Dbs.db[i].dbCfg.precision, - superTblInfo); - } + init_rand_data(); + + // create database and super tables + if(createDatabasesAndStables() != 0) { + if (g_fpOfInsertResult) + fclose(g_fpOfInsertResult); + return -1; + } + + // pretreatement + if (prepareSampleData() != 0) { + if (g_fpOfInsertResult) + fclose(g_fpOfInsertResult); + return -1; + } + + double start; + double end; + + // create child tables + start = taosGetTimestampMs(); + createChildTables(); + end = taosGetTimestampMs(); + + if (g_totalChildTables > 0) { + fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n", + (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl); + if (g_fpOfInsertResult) { + fprintf(g_fpOfInsertResult, + "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n", + (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl); } - } - } else { - startMultiThreadInsertData( - g_Dbs.threadCount, - g_Dbs.db[i].dbName, - g_Dbs.db[i].dbCfg.precision, - NULL); } - } - //end = taosGetTimestampMs(); - //int64_t totalInsertRows = 0; - //int64_t totalAffectedRows = 0; - //for (int i = 0; i < g_Dbs.dbCount; i++) { - // for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - // totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows; - // totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows; - //} - //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount); - postFreeResource(); + // create sub threads for inserting data + //start = taosGetTimestampMs(); + for (int i = 0; i < g_Dbs.dbCount; i++) { + if (g_Dbs.use_metric) { + if (g_Dbs.db[i].superTblCount > 0) { + for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { - return 0; + SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j]; + + if (superTblInfo && (superTblInfo->insertRows > 0)) { + startMultiThreadInsertData( + g_Dbs.threadCount, + g_Dbs.db[i].dbName, + g_Dbs.db[i].dbCfg.precision, + superTblInfo); + } + } + } + } else { + startMultiThreadInsertData( + g_Dbs.threadCount, + g_Dbs.db[i].dbName, + g_Dbs.db[i].dbCfg.precision, + NULL); + } + } + //end = taosGetTimestampMs(); + + //int64_t totalInsertRows = 0; + //int64_t totalAffectedRows = 0; + //for (int i = 0; i < g_Dbs.dbCount; i++) { + // for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + // totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows; + // totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows; + //} + //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount); + postFreeResource(); + + return 0; } static void *specifiedTableQuery(void *sarg) { @@ -7954,116 +7987,116 @@ static void initOfQueryMeta() { } static void setParaFromArg(){ - if (g_args.host) { - tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE); - } else { - tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); - } + if (g_args.host) { + tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE); + } else { + tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); + } - if (g_args.user) { - tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE); - } + if (g_args.user) { + tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE); + } - if (g_args.password) { - tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE); - } + if (g_args.password) { + tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE); + } - if (g_args.port) { - g_Dbs.port = g_args.port; - } + if (g_args.port) { + g_Dbs.port = g_args.port; + } - g_Dbs.threadCount = g_args.num_of_threads; - g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; - - g_Dbs.dbCount = 1; - g_Dbs.db[0].drop = true; - - tstrncpy(g_Dbs.db[0].dbName, g_args.database, TSDB_DB_NAME_LEN); - g_Dbs.db[0].dbCfg.replica = g_args.replica; - tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", 8); - - tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN); - - g_Dbs.use_metric = g_args.use_metric; - g_Dbs.insert_only = g_args.insert_only; - - g_Dbs.do_aggreFunc = true; - - char dataString[STRING_LEN]; - char **data_type = g_args.datatype; - - memset(dataString, 0, STRING_LEN); - - if (strcasecmp(data_type[0], "BINARY") == 0 - || strcasecmp(data_type[0], "BOOL") == 0 - || strcasecmp(data_type[0], "NCHAR") == 0 ) { - g_Dbs.do_aggreFunc = false; - } - - if (g_args.use_metric) { - g_Dbs.db[0].superTblCount = 1; - tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", TSDB_TABLE_NAME_LEN); - g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; g_Dbs.threadCount = g_args.num_of_threads; g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; - g_Dbs.asyncMode = g_args.async_mode; - g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; - g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; - g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; - g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; - tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, - g_args.tb_prefix, TSDB_TABLE_NAME_LEN - 20); - tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); + g_Dbs.dbCount = 1; + g_Dbs.db[0].drop = true; - if (g_args.iface == INTERFACE_BUT) { - g_Dbs.db[0].superTbls[0].iface = TAOSC_IFACE; - } else { - g_Dbs.db[0].superTbls[0].iface = g_args.iface; - } - tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, - "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP; + tstrncpy(g_Dbs.db[0].dbName, g_args.database, TSDB_DB_NAME_LEN); + g_Dbs.db[0].dbCfg.replica = g_args.replica; + tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", 8); - g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT; - g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len; + tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN); - g_Dbs.db[0].superTbls[0].columnCount = 0; - for (int i = 0; i < MAX_NUM_COLUMNS; i++) { - if (data_type[i] == NULL) { - break; - } + g_Dbs.use_metric = g_args.use_metric; + g_Dbs.insert_only = g_args.insert_only; - tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, - data_type[i], strlen(data_type[i]) + 1); - g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; - g_Dbs.db[0].superTbls[0].columnCount++; + g_Dbs.do_aggreFunc = true; + + char dataString[STRING_LEN]; + char **data_type = g_args.datatype; + + memset(dataString, 0, STRING_LEN); + + if (strcasecmp(data_type[0], "BINARY") == 0 + || strcasecmp(data_type[0], "BOOL") == 0 + || strcasecmp(data_type[0], "NCHAR") == 0 ) { + g_Dbs.do_aggreFunc = false; } - if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) { - g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR; - } else { - for (int i = g_Dbs.db[0].superTbls[0].columnCount; - i < g_args.num_of_CPR; i++) { - tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, + if (g_args.use_metric) { + g_Dbs.db[0].superTblCount = 1; + tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", TSDB_TABLE_NAME_LEN); + g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; + g_Dbs.threadCount = g_args.num_of_threads; + g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; + g_Dbs.asyncMode = g_args.async_mode; + + g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; + g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; + g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; + g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; + tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, + g_args.tb_prefix, TSDB_TABLE_NAME_LEN - 20); + tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); + + if (g_args.iface == INTERFACE_BUT) { + g_Dbs.db[0].superTbls[0].iface = TAOSC_IFACE; + } else { + g_Dbs.db[0].superTbls[0].iface = g_args.iface; + } + tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, + "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP; + + g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT; + g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len; + + g_Dbs.db[0].superTbls[0].columnCount = 0; + for (int i = 0; i < MAX_NUM_COLUMNS; i++) { + if (data_type[i] == NULL) { + break; + } + + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, + data_type[i], strlen(data_type[i]) + 1); + g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; + g_Dbs.db[0].superTbls[0].columnCount++; + } + + if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) { + g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR; + } else { + for (int i = g_Dbs.db[0].superTbls[0].columnCount; + i < g_args.num_of_CPR; i++) { + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, + "INT", strlen("INT") + 1); + g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; + g_Dbs.db[0].superTbls[0].columnCount++; + } + } + + tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", strlen("INT") + 1); - g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; - g_Dbs.db[0].superTbls[0].columnCount++; - } + g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; + + tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, + "BINARY", strlen("BINARY") + 1); + g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; + g_Dbs.db[0].superTbls[0].tagCount = 2; + } else { + g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; + g_Dbs.db[0].superTbls[0].tagCount = 0; } - - tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, - "INT", strlen("INT") + 1); - g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; - - tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, - "BINARY", strlen("BINARY") + 1); - g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; - g_Dbs.db[0].superTbls[0].tagCount = 2; - } else { - g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; - g_Dbs.db[0].superTbls[0].tagCount = 0; - } } /* Function to do regular expression check */ diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index c76a00b8cf..6e5cf14b96 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1503,6 +1503,18 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg) { return TSDB_CODE_MND_FIELD_NOT_EXIST; } + // check exceed max row bytes + int32_t i; + uint32_t nLen = 0; + for (i = 0; i < pStable->numOfColumns; ++i) { + nLen += (pStable->schema[i].colId == col) ? pAlter->schema[0].bytes : pStable->schema[i].bytes; + } + if (nLen > TSDB_MAX_BYTES_PER_ROW) { + mError("msg:%p, app:%p stable:%s, change column, name:%s exceed max row bytes", pMsg, pMsg->rpcMsg.ahandle, + pStable->info.tableId, name); + return TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES; + } + // update SSchema *schema = (SSchema *) (pStable->schema + col); ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR); @@ -3091,6 +3103,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { int32_t len = tsCompressString(pMultiMeta->meta, dataLen, 1, tmp + sizeof(SMultiTableMeta), (int32_t)dataLen + 2, ONE_STAGE_COMP, NULL, 0); + pMultiMeta->metaClone = pInfo->metaClone; pMultiMeta->rawLen = pMultiMeta->contLen; if (len == -1 || len >= dataLen + 2) { // compress failed, do not compress this binary data pMultiMeta->compressed = 0; diff --git a/src/query/inc/qExtbuffer.h b/src/query/inc/qExtbuffer.h index b851fbb3e0..cf0e8ce31a 100644 --- a/src/query/inc/qExtbuffer.h +++ b/src/query/inc/qExtbuffer.h @@ -77,13 +77,13 @@ typedef struct tFilePagesItem { typedef struct SSchemaEx { struct SSchema field; - int16_t offset; + int32_t offset; } SSchemaEx; typedef struct SColumnModel { int32_t capacity; int32_t numOfCols; - int16_t rowSize; + int32_t rowSize; SSchemaEx *pFields; } SColumnModel; diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index b31a93fd2d..d2802d9fe0 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -66,7 +66,8 @@ static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int return pResultRowInfo->pResult[slot]; } -static FORCE_INLINE char *getPosInResultPage(SQueryAttr *pQueryAttr, tFilePage* page, int32_t rowOffset, int16_t offset) { +static FORCE_INLINE char* getPosInResultPage(SQueryAttr* pQueryAttr, tFilePage* page, int32_t rowOffset, + int32_t offset) { assert(rowOffset >= 0 && pQueryAttr != NULL); int32_t numOfRows = (int32_t)GET_ROW_PARAM_FOR_MULTIOUTPUT(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery); diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 93e6de3821..d96b260b13 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -3982,7 +3982,7 @@ void blockInfo_func(SQLFunctionCtx* pCtx) { int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist); - pDist->rowSize = (int16_t) pCtx->param[0].i64; + pDist->rowSize = (uint16_t)pCtx->param[0].i64; memcpy(pCtx->pOutput, pCtx->pInput, sizeof(int32_t) + len); @@ -4129,7 +4129,7 @@ void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); - pDist->rowSize = (int16_t)pCtx->param[0].i64; + pDist->rowSize = (uint16_t)pCtx->param[0].i64; generateBlockDistResult(pDist, pCtx->pOutput); // cannot set the numOfIteratedElems again since it is set during previous iteration diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 0f3d56515d..288a206491 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3583,7 +3583,7 @@ void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pRe // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); - int16_t offset = 0; + int32_t offset = 0; for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); @@ -3897,7 +3897,7 @@ static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pRow->pageId); - int16_t offset = 0; + int32_t offset = 0; for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, j); int32_t bytes = pColInfoData->info.bytes; diff --git a/src/query/src/qSqlParser.c b/src/query/src/qSqlParser.c index af59a5e417..874ec7b692 100644 --- a/src/query/src/qSqlParser.c +++ b/src/query/src/qSqlParser.c @@ -160,7 +160,8 @@ tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType) { pSqlExpr->type = SQL_NODE_VALUE; pSqlExpr->flags |= 1 << EXPR_FLAG_NS_TIMESTAMP; } else if (optrType == TK_VARIABLE) { - // use nanosecond by default TODO set value after getting database precision + // use nanosecond by default + // TODO set value after getting database precision int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->value.i64, TSDB_TIME_PRECISION_NANO); if (ret != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 27c877fafa..825b7960de 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -267,6 +267,10 @@ static void writeDataToDisk(STSBuf* pTSBuf) { if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); + } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { + metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + float tfloat = (float)pBlock->tag.dKey; + metaLen += (int32_t)fwrite(&tfloat, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); metaLen += (int32_t)fwrite(&pBlock->tag.i64, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); @@ -351,6 +355,11 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); UNUSED(sz); + } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { + float tfloat = 0; + sz = fread(&tfloat, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); + pBlock->tag.dKey = (double)tfloat; + UNUSED(sz); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value sz = fread(&pBlock->tag.i64, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); UNUSED(sz); diff --git a/src/tsdb/inc/tsdbMemTable.h b/src/tsdb/inc/tsdbMemTable.h index babb7024b2..67e9976c70 100644 --- a/src/tsdb/inc/tsdbMemTable.h +++ b/src/tsdb/inc/tsdbMemTable.h @@ -71,27 +71,27 @@ int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxK TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo); void* tsdbCommitData(STsdbRepo* pRepo); -static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { +static FORCE_INLINE SMemRow tsdbNextIterRow(SSkipListIterator* pIter) { if (pIter == NULL) return NULL; SSkipListNode* node = tSkipListIterGet(pIter); if (node == NULL) return NULL; - return (SDataRow)SL_GET_NODE_DATA(node); + return (SMemRow)SL_GET_NODE_DATA(node); } static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) { - SDataRow row = tsdbNextIterRow(pIter); + SMemRow row = tsdbNextIterRow(pIter); if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL; - return dataRowKey(row); + return memRowKey(row); } static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) { - SDataRow row = tsdbNextIterRow(pIter); + SMemRow row = tsdbNextIterRow(pIter); if (row == NULL) return TKEY_NULL; - return dataRowTKey(row); + return memRowTKey(row); } #endif /* _TD_TSDB_MEMTABLE_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbMeta.h b/src/tsdb/inc/tsdbMeta.h index 45bbd5a7c6..9a8de01f71 100644 --- a/src/tsdb/inc/tsdbMeta.h +++ b/src/tsdb/inc/tsdbMeta.h @@ -32,7 +32,7 @@ typedef struct STable { void* eventHandler; // TODO void* streamHandler; // TODO TSKEY lastKey; - SDataRow lastRow; + SMemRow lastRow; char* sql; void* cqhandle; SRWLatch latch; // TODO: implementa latch functions @@ -148,7 +148,7 @@ static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) { } static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) { - ASSERT(pTable->lastRow == NULL || pTable->lastKey == dataRowKey(pTable->lastRow)); + ASSERT((pTable->lastRow == NULL) || (pTable->lastKey == memRowKey(pTable->lastRow))); return pTable->lastKey; } diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 75b072b063..6330da6058 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -920,7 +920,8 @@ int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCo SDataCol * pDataCol = pDataCols->cols + ncol; SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; - if (isNEleNull(pDataCol, rowsToWrite)) { // all data to commit are NULL, just ignore it + // if (isNEleNull(pDataCol, rowsToWrite)) { // all data to commit are NULL, just ignore it + if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it continue; } @@ -1264,12 +1265,12 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt while (true) { key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); bool isRowDel = false; - SDataRow row = tsdbNextIterRow(pCommitIter->pIter); - if (row == NULL || dataRowKey(row) > maxKey) { + SMemRow row = tsdbNextIterRow(pCommitIter->pIter); + if (row == NULL || memRowKey(row) > maxKey) { key2 = INT64_MAX; } else { - key2 = dataRowKey(row); - isRowDel = dataRowDeleted(row); + key2 = memRowKey(row); + isRowDel = memRowDeleted(row); } if (key1 == INT64_MAX && key2 == INT64_MAX) break; @@ -1284,24 +1285,24 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt (*iter)++; } else if (key1 > key2) { if (!isRowDel) { - if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); + if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row)); ASSERT(pSchema != NULL); } - tdAppendDataRowToDataCol(row, pSchema, pTarget); + tdAppendMemRowToDataCol(row, pSchema, pTarget); } tSkipListIterNext(pCommitIter->pIter); } else { if (update) { if (!isRowDel) { - if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); + if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row)); ASSERT(pSchema != NULL); } - tdAppendDataRowToDataCol(row, pSchema, pTarget); + tdAppendMemRowToDataCol(row, pSchema, pTarget); } } else { ASSERT(!isRowDel); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 09f3585b9d..8266a7c20f 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -641,7 +641,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea int numColumns; int32_t blockIdx; SDataStatis* pBlockStatis = NULL; - SDataRow row = NULL; + SMemRow row = NULL; // restore last column data with last schema int err = 0; @@ -657,13 +657,15 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea } } - row = taosTMalloc(dataRowMaxBytesFromSchema(pSchema)); + row = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); if (row == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; err = -1; goto out; } - tdInitDataRow(row, pSchema); + + memRowSetType(row, SMEM_ROW_DATA); + tdInitDataRow(memRowDataBody(row), pSchema); // first load block index info if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { @@ -720,9 +722,9 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea // OK,let's load row from backward to get not-null column for (int32_t rowId = pBlock->numOfRows - 1; rowId >= 0; rowId--) { SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; - tdAppendColVal(row, tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->bytes, pCol->offset); + tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->offset); //SDataCol *pDataCol = readh.pDCols[0]->cols + j; - void* value = tdGetRowDataOfCol(row, (int8_t)pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); + void *value = tdGetRowDataOfCol(memRowDataBody(row), (int8_t)pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); if (isNull(value, pCol->type)) { continue; } @@ -742,8 +744,8 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea // save row ts(in column 0) pDataCol = pReadh->pDCols[0]->cols + 0; pCol = schemaColAt(pSchema, 0); - tdAppendColVal(row, tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->bytes, pCol->offset); - pLastCol->ts = dataRowKey(row); + tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->offset); + pLastCol->ts = memRowKey(row); pTable->restoreColumnNum += 1; @@ -779,18 +781,18 @@ static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, // Get the data in row STSchema *pSchema = tsdbGetTableSchema(pTable); - pTable->lastRow = taosTMalloc(dataRowMaxBytesFromSchema(pSchema)); + pTable->lastRow = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); if (pTable->lastRow == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } - - tdInitDataRow(pTable->lastRow, pSchema); + memRowSetType(pTable->lastRow, SMEM_ROW_DATA); + tdInitDataRow(memRowDataBody(pTable->lastRow), pSchema); for (int icol = 0; icol < schemaNCols(pSchema); icol++) { STColumn *pCol = schemaColAt(pSchema, icol); SDataCol *pDataCol = pReadh->pDCols[0]->cols + icol; - tdAppendColVal(pTable->lastRow, tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, pCol->bytes, - pCol->offset); + tdAppendColVal(memRowDataBody(pTable->lastRow), tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, + pCol->offset); } return 0; diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 50b5d32177..9ac5503e5f 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -21,7 +21,7 @@ typedef struct { int32_t totalLen; int32_t len; - SDataRow row; + SMemRow row; } SSubmitBlkIter; typedef struct { @@ -36,20 +36,19 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable); static void tsdbFreeTableData(STableData *pTableData); static char * tsdbGetTsTupleKey(const void *data); static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables); -static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row); +static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SMemRow row); static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter); -static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter); +static SMemRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter); static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg); static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows); -static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void **ppRow); +static int tsdbCopyRowToMem(STsdbRepo *pRepo, SMemRow row, STable *pTable, void **ppRow); static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter); static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock); static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable); static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **rows, int rowCounter); static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter); -static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow row); - -static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey, +static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow row); +static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SMemRow row, TSKEY minKey, TSKEY maxKey, TSKEY now); int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) { @@ -354,7 +353,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey TSKEY fKey = 0; bool isRowDel = false; int filterIter = 0; - SDataRow row = NULL; + SMemRow row = NULL; SMergeInfo mInfo; if (pMergeInfo == NULL) pMergeInfo = &mInfo; @@ -365,12 +364,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (pCols) tdResetDataCols(pCols); row = tsdbNextIterRow(pIter); - if (row == NULL || dataRowKey(row) > maxKey) { + if (row == NULL || memRowKey(row) > maxKey) { rowKey = INT64_MAX; isRowDel = false; } else { - rowKey = dataRowKey(row); - isRowDel = dataRowDeleted(row); + rowKey = memRowKey(row); + isRowDel = memRowDeleted(row); } if (filterIter >= nFilterKeys) { @@ -407,12 +406,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey tSkipListIterNext(pIter); row = tsdbNextIterRow(pIter); - if (row == NULL || dataRowKey(row) > maxKey) { + if (row == NULL || memRowKey(row) > maxKey) { rowKey = INT64_MAX; isRowDel = false; } else { - rowKey = dataRowKey(row); - isRowDel = dataRowDeleted(row); + rowKey = memRowKey(row); + isRowDel = memRowDeleted(row); } } else { if (isRowDel) { @@ -437,12 +436,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey tSkipListIterNext(pIter); row = tsdbNextIterRow(pIter); - if (row == NULL || dataRowKey(row) > maxKey) { + if (row == NULL || memRowKey(row) > maxKey) { rowKey = INT64_MAX; isRowDel = false; } else { - rowKey = dataRowKey(row); - isRowDel = dataRowDeleted(row); + rowKey = memRowKey(row); + isRowDel = memRowDeleted(row); } filterIter++; @@ -548,7 +547,7 @@ static void tsdbFreeTableData(STableData *pTableData) { } } -static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); } +static char *tsdbGetTsTupleKey(const void *data) { return memRowTuple((SMemRow)data); } static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { ASSERT(pMemTable->maxTables < maxTables); @@ -572,17 +571,17 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { return 0; } -static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row) { +static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SMemRow row) { if (pCols) { - if (*ppSchema == NULL || schemaVersion(*ppSchema) != dataRowVersion(row)) { - *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row)); + if (*ppSchema == NULL || schemaVersion(*ppSchema) != memRowVersion(row)) { + *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, memRowVersion(row)); if (*ppSchema == NULL) { ASSERT(false); return -1; } } - tdAppendDataRowToDataCol(row, *ppSchema, pCols); + tdAppendMemRowToDataCol(row, *ppSchema, pCols); } return 0; @@ -592,31 +591,32 @@ static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { if (pBlock->dataLen <= 0) return -1; pIter->totalLen = pBlock->dataLen; pIter->len = 0; - pIter->row = (SDataRow)(pBlock->data+pBlock->schemaLen); + pIter->row = (SMemRow)(pBlock->data + pBlock->schemaLen); return 0; } -static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) { - SDataRow row = pIter->row; +static SMemRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) { + SMemRow row = pIter->row; // firstly, get current row if (row == NULL) return NULL; - pIter->len += dataRowLen(row); - if (pIter->len >= pIter->totalLen) { + pIter->len += memRowTLen(row); + if (pIter->len >= pIter->totalLen) { // reach the end pIter->row = NULL; } else { - pIter->row = (char *)row + dataRowLen(row); + pIter->row = (char *)row + memRowTLen(row); // secondly, move to next row } return row; } -static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey, +static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SMemRow row, TSKEY minKey, TSKEY maxKey, TSKEY now) { - if (dataRowKey(row) < minKey || dataRowKey(row) > maxKey) { + TSKEY rowKey = memRowKey(row); + if (rowKey < minKey || rowKey > maxKey) { tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 " maxKey %" PRId64 " row key %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey, - dataRowKey(row)); + rowKey); terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; return -1; } @@ -630,7 +630,7 @@ static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) { SSubmitMsgIter msgIter = {0}; SSubmitBlk * pBlock = NULL; SSubmitBlkIter blkIter = {0}; - SDataRow row = NULL; + SMemRow row = NULL; TSKEY now = taosGetTimestamp(pRepo->config.precision); TSKEY minKey = now - tsTickPerDay[pRepo->config.precision] * pRepo->config.keep; TSKEY maxKey = now + tsTickPerDay[pRepo->config.precision] * pRepo->config.daysPerFile; @@ -698,7 +698,7 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t * int64_t points = 0; STable * pTable = NULL; SSubmitBlkIter blkIter = {0}; - SDataRow row = NULL; + SMemRow row = NULL; void * rows[TSDB_MAX_INSERT_BATCH] = {0}; int rowCounter = 0; @@ -744,10 +744,10 @@ _err: return -1; } -static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void **ppRow) { +static int tsdbCopyRowToMem(STsdbRepo *pRepo, SMemRow row, STable *pTable, void **ppRow) { STsdbCfg * pCfg = &pRepo->config; - TKEY tkey = dataRowTKey(row); - TSKEY key = dataRowKey(row); + TKEY tkey = memRowTKey(row); + TSKEY key = memRowKey(row); bool isRowDelete = TKEY_IS_DELETED(tkey); if (isRowDelete) { @@ -765,15 +765,15 @@ static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void } } - void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row)); + void *pRow = tsdbAllocBytes(pRepo, memRowTLen(row)); if (pRow == NULL) { - tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s", - REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno)); + tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %" PRIu32 " bytes since %s", + REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), memRowTLen(row), tstrerror(terrno)); return -1; } - dataRowCpy(pRow, row); - ppRow[0] = pRow; + memRowCpy(pRow, row); + ppRow[0] = pRow; // save the memory address of data rows tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), isRowDelete ? "deleted from" : "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), @@ -932,13 +932,15 @@ static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **ro int64_t osize = SL_SIZE(pTableData->pData); tSkipListPutBatch(pTableData->pData, rows, rowCounter); int64_t dsize = SL_SIZE(pTableData->pData) - osize; + TSKEY keyFirstRow = memRowKey(rows[0]); + TSKEY keyLastRow = memRowKey(rows[rowCounter - 1]); - if (pMemTable->keyFirst > dataRowKey(rows[0])) pMemTable->keyFirst = dataRowKey(rows[0]); - if (pMemTable->keyLast < dataRowKey(rows[rowCounter - 1])) pMemTable->keyLast = dataRowKey(rows[rowCounter - 1]); + if (pMemTable->keyFirst > keyFirstRow) pMemTable->keyFirst = keyFirstRow; + if (pMemTable->keyLast < keyLastRow) pMemTable->keyLast = keyLastRow; pMemTable->numOfRows += dsize; - if (pTableData->keyFirst > dataRowKey(rows[0])) pTableData->keyFirst = dataRowKey(rows[0]); - if (pTableData->keyLast < dataRowKey(rows[rowCounter - 1])) pTableData->keyLast = dataRowKey(rows[rowCounter - 1]); + if (pTableData->keyFirst > keyFirstRow) pTableData->keyFirst = keyFirstRow; + if (pTableData->keyLast < keyLastRow) pTableData->keyLast = keyLastRow; pTableData->numOfRows += dsize; // update table latest info @@ -954,8 +956,8 @@ static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) { STsdbBufPool *pBufPool = pRepo->pPool; for (int i = rowCounter - 1; i >= 0; --i) { - SDataRow row = (SDataRow)rows[i]; - int bytes = (int)dataRowLen(row); + SMemRow row = (SMemRow)rows[i]; + int bytes = (int)memRowTLen(row); if (pRepo->mem->extraBuffList == NULL) { STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); @@ -988,21 +990,23 @@ static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) { } } -static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SDataRow row) { - tsdbDebug("vgId:%d updateTableLatestColumn, %s row version:%d", REPO_ID(pRepo), pTable->name->data, dataRowVersion(row)); +static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow row) { + tsdbDebug("vgId:%d updateTableLatestColumn, %s row version:%d", REPO_ID(pRepo), pTable->name->data, + memRowVersion(row)); STSchema* pSchema = tsdbGetTableLatestSchema(pTable); if (tsdbUpdateLastColSchema(pTable, pSchema) < 0) { return; } - pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); + pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); if (pSchema == NULL) { return; } SDataCol *pLatestCols = pTable->lastCols; + bool isDataRow = isDataRow(row); for (int16_t j = 0; j < schemaNCols(pSchema); j++) { STColumn *pTCol = schemaColAt(pSchema, j); // ignore not exist colId @@ -1010,9 +1014,21 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SDataRow r if (idx == -1) { continue; } - - void* value = tdGetRowDataOfCol(row, (int8_t)pTCol->type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); - if (isNull(value, pTCol->type)) { + + void *value = NULL; + + if (isDataRow) { + value = tdGetRowDataOfCol(memRowDataBody(row), (int8_t)pTCol->type, + TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); + } else { + // SKVRow + SColIdx *pColIdx = tdGetKVRowIdxOfCol(memRowKvBody(row), pTCol->colId); + if (pColIdx) { + value = tdGetKvRowDataOfCol(memRowKvBody(row), pColIdx->offset); + } + } + + if ((value == NULL) || isNull(value, pTCol->type)) { continue; } @@ -1027,11 +1043,11 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SDataRow r memcpy(pDataCol->pData, value, pDataCol->bytes); //tsdbInfo("updateTableLatestColumn vgId:%d cache column %d for %d,%s", REPO_ID(pRepo), j, pDataCol->bytes, (char*)pDataCol->pData); - pDataCol->ts = dataRowKey(row); + pDataCol->ts = memRowKey(row); } } -static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow row) { +static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow row) { STsdbCfg *pCfg = &pRepo->config; // if cacheLastRow config has been reset, free the lastRow @@ -1042,31 +1058,31 @@ static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow TSDB_WUNLOCK_TABLE(pTable); } - if (tsdbGetTableLastKeyImpl(pTable) < dataRowKey(row)) { + if (tsdbGetTableLastKeyImpl(pTable) < memRowKey(row)) { if (CACHE_LAST_ROW(pCfg) || pTable->lastRow != NULL) { - SDataRow nrow = pTable->lastRow; - if (taosTSizeof(nrow) < dataRowLen(row)) { - SDataRow orow = nrow; - nrow = taosTMalloc(dataRowLen(row)); + SMemRow nrow = pTable->lastRow; + if (taosTSizeof(nrow) < memRowTLen(row)) { + SMemRow orow = nrow; + nrow = taosTMalloc(memRowTLen(row)); if (nrow == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } - dataRowCpy(nrow, row); + memRowCpy(nrow, row); TSDB_WLOCK_TABLE(pTable); - pTable->lastKey = dataRowKey(row); + pTable->lastKey = memRowKey(row); pTable->lastRow = nrow; TSDB_WUNLOCK_TABLE(pTable); taosTZfree(orow); } else { TSDB_WLOCK_TABLE(pTable); - pTable->lastKey = dataRowKey(row); - dataRowCpy(nrow, row); + pTable->lastKey = memRowKey(row); + memRowCpy(nrow, row); TSDB_WUNLOCK_TABLE(pTable); } } else { - pTable->lastKey = dataRowKey(row); + pTable->lastKey = memRowKey(row); } if (CACHE_LAST_NULL_COLUMN(pCfg)) { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 1eafb5e233..c48851df45 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -147,7 +147,7 @@ typedef struct STableGroupSupporter { static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList); static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList); static int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle); -static int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey); +static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey); static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); @@ -734,8 +734,8 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); assert(node != NULL); - SDataRow row = (SDataRow)SL_GET_NODE_DATA(node); - TSKEY key = dataRowKey(row); // first timestamp in buffer + SMemRow row = (SMemRow)SL_GET_NODE_DATA(node); + TSKEY key = memRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pMem->keyFirst, pMem->keyLast, @@ -756,8 +756,8 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); assert(node != NULL); - SDataRow row = (SDataRow)SL_GET_NODE_DATA(node); - TSKEY key = dataRowKey(row); // first timestamp in buffer + SMemRow row = (SMemRow)SL_GET_NODE_DATA(node); + TSKEY key = memRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast, @@ -781,19 +781,19 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { tSkipListDestroyIter(pCheckInfo->iiter); } -static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) { - SDataRow rmem = NULL, rimem = NULL; +static SMemRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) { + SMemRow rmem = NULL, rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); if (node != NULL) { - rmem = (SDataRow)SL_GET_NODE_DATA(node); + rmem = (SMemRow)SL_GET_NODE_DATA(node); } } if (pCheckInfo->iiter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); if (node != NULL) { - rimem = (SDataRow)SL_GET_NODE_DATA(node); + rimem = (SMemRow)SL_GET_NODE_DATA(node); } } @@ -811,8 +811,8 @@ static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order return rimem; } - TSKEY r1 = dataRowKey(rmem); - TSKEY r2 = dataRowKey(rimem); + TSKEY r1 = memRowKey(rmem); + TSKEY r2 = memRowKey(rimem); if (r1 == r2) { // data ts are duplicated, ignore the data in mem if (!update) { @@ -891,12 +891,12 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { initTableMemIterator(pHandle, pCheckInfo); } - SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order, pCfg->update); + SMemRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order, pCfg->update); if (row == NULL) { return false; } - pCheckInfo->lastKey = dataRowKey(row); // first timestamp in buffer + pCheckInfo->lastKey = memRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, 0x%"PRIx64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qId); @@ -1155,11 +1155,11 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p int32_t code = TSDB_CODE_SUCCESS; /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo); - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); + SMemRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); assert(cur->pos >= 0 && cur->pos <= binfo.rows); - TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL; + TSKEY key = (row != NULL) ? memRowKey(row) : TSKEY_INITIAL_VAL; if (key != TSKEY_INITIAL_VAL) { tsdbDebug("%p key in mem:%"PRId64", 0x%"PRIx64, pQueryHandle, key, pQueryHandle->qId); } else { @@ -1401,7 +1401,7 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity // todo refactor, only copy one-by-one for (int32_t k = start; k < num + start; ++k) { - char* p = tdGetColDataOfRow(src, k); + const char* p = tdGetColDataOfRow(src, k); memcpy(dst, p, varDataTLen(p)); dst += bytes; } @@ -1452,88 +1452,170 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity return numOfRows + num; } -static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SDataRow row, +static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SMemRow row, int32_t numOfCols, STable* pTable, STSchema* pSchema) { char* pData = NULL; - // the schema version info is embeded in SDataRow + // the schema version info is embedded in SDataRow, and use latest schema version for SKVRow int32_t numOfRowCols = 0; if (pSchema == NULL) { - pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); + pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); numOfRowCols = schemaNCols(pSchema); } else { numOfRowCols = schemaNCols(pSchema); } - - int32_t i = 0, j = 0; - while(i < numOfCols && j < numOfRowCols) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (pSchema->columns[j].colId < pColInfo->info.colId) { - j++; - continue; - } - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - } else { - pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; - } + int32_t i = 0; - if (pSchema->columns[j].colId == pColInfo->info.colId) { - void* value = tdGetRowDataOfCol(row, (int8_t)pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); - switch (pColInfo->info.type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - memcpy(pData, value, varDataTLen(value)); - break; - case TSDB_DATA_TYPE_NULL: - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *)pData = *(uint8_t *)value; - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *(uint16_t *)pData = *(uint16_t *)value; - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - *(uint32_t *)pData = *(uint32_t *)value; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - *(uint64_t *)pData = *(uint64_t *)value; - break; - case TSDB_DATA_TYPE_FLOAT: - SET_FLOAT_PTR(pData, value); - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_PTR(pData, value); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - *(TSKEY *)pData = tdGetKey(*(TKEY *)value); - } else { - *(TSKEY *)pData = *(TSKEY *)value; - } - break; - default: - memcpy(pData, value, pColInfo->info.bytes); + if (isDataRow(row)) { + SDataRow dataRow = memRowDataBody(row); + int32_t j = 0; + while (i < numOfCols && j < numOfRowCols) { + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + if (pSchema->columns[j].colId < pColInfo->info.colId) { + j++; + continue; } - j++; - i++; - } else { // pColInfo->info.colId < pSchema->columns[j].colId, it is a NULL data + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; + } else { + pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; + } + + if (pSchema->columns[j].colId == pColInfo->info.colId) { + void* value = + tdGetRowDataOfCol(dataRow, (int8_t)pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); + switch (pColInfo->info.type) { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + memcpy(pData, value, varDataTLen(value)); + break; + case TSDB_DATA_TYPE_NULL: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + *(uint8_t*)pData = *(uint8_t*)value; + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + *(uint16_t*)pData = *(uint16_t*)value; + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + *(uint32_t*)pData = *(uint32_t*)value; + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + *(uint64_t*)pData = *(uint64_t*)value; + break; + case TSDB_DATA_TYPE_FLOAT: + SET_FLOAT_PTR(pData, value); + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_PTR(pData, value); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + *(TSKEY*)pData = tdGetKey(*(TKEY*)value); + } else { + *(TSKEY*)pData = *(TSKEY*)value; + } + break; + default: + memcpy(pData, value, pColInfo->info.bytes); + } + + j++; + i++; + } else { // pColInfo->info.colId < pSchema->columns[j].colId, it is a NULL data + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { + setVardataNull(pData, pColInfo->info.type); + } else { + setNull(pData, pColInfo->info.type, pColInfo->info.bytes); + } + i++; + } + } + } else if (isKvRow(row)) { + SKVRow kvRow = memRowKvBody(row); + int32_t k = 0; + int32_t nKvRowCols = kvRowNCols(kvRow); + + while (i < numOfCols && k < nKvRowCols) { + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColIdx* pColIdx = kvRowColIdxAt(kvRow, k); + + if (pColIdx->colId < pColInfo->info.colId) { + ++k; + continue; + } + + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; + } else { + pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; + } + + if (pColIdx->colId == pColInfo->info.colId) { + // offset of pColIdx for SKVRow including the TD_KV_ROW_HEAD_SIZE + void* value = tdGetKvRowDataOfCol(kvRow, pColIdx->offset); + switch (pColInfo->info.type) { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + memcpy(pData, value, varDataTLen(value)); + break; + case TSDB_DATA_TYPE_NULL: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + *(uint8_t*)pData = *(uint8_t*)value; + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + *(uint16_t*)pData = *(uint16_t*)value; + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + *(uint32_t*)pData = *(uint32_t*)value; + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + *(uint64_t*)pData = *(uint64_t*)value; + break; + case TSDB_DATA_TYPE_FLOAT: + SET_FLOAT_PTR(pData, value); + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_PTR(pData, value); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + *(TSKEY*)pData = tdGetKey(*(TKEY*)value); + } else { + *(TSKEY*)pData = *(TSKEY*)value; + } + break; + default: + memcpy(pData, value, pColInfo->info.bytes); + } + ++k; + ++i; + continue; + } + // If (pColInfo->info.colId < pColIdx->colId), it is NULL data if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { setVardataNull(pData, pColInfo->info.type); } else { setNull(pData, pColInfo->info.type, pColInfo->info.bytes); } - i++; + ++i; } + } else { + ASSERT(0); } - while (i < numOfCols) { // the remain columns are all null data + while (i < numOfCols) { // the remain columns are all null data SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); if (ASCENDING_TRAVERSE(pQueryHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; @@ -1550,7 +1632,6 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, i++; } } - static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols) { if (numOfRows == 0 || ASCENDING_TRAVERSE(pQueryHandle->order)) { return; @@ -1730,12 +1811,12 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; do { - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); + SMemRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); if (row == NULL) { break; } - TSKEY key = dataRowKey(row); + TSKEY key = memRowKey(row); if ((key > pQueryHandle->window.ekey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < pQueryHandle->window.ekey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { break; @@ -1748,11 +1829,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - if (rv != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); - rv = dataRowVersion(row); - } - + if (rv != memRowVersion(row)) { + pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); + rv = memRowVersion(row); + } + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable, pSchema); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { @@ -1766,11 +1847,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* moveToNextRowInMem(pCheckInfo); } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it if (pCfg->update) { - if (rv != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); - rv = dataRowVersion(row); + if (rv != memRowVersion(row)) { + pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); + rv = memRowVersion(row); } - + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable, pSchema); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { @@ -1820,8 +1901,10 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* * copy them all to result buffer, since it may be overlapped with file data block. */ if (node == NULL || - ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && + ASCENDING_TRAVERSE(pQueryHandle->order)) || + ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && + !ASCENDING_TRAVERSE(pQueryHandle->order))) { // no data in cache or data in cache is greater than the ekey of time window, load data from file block if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = tsArray[pos]; @@ -2407,12 +2490,12 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int STSchema* pSchema = NULL; do { - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); + SMemRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); if (row == NULL) { break; } - TSKEY key = dataRowKey(row); + TSKEY key = memRowKey(row); if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, pQueryHandle->window.ekey); @@ -2425,9 +2508,9 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int } win->ekey = key; - if (rv != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); - rv = dataRowVersion(row); + if (rv != memRowVersion(row)) { + pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); + rv = memRowVersion(row); } copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable, pSchema); @@ -2544,7 +2627,7 @@ static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) { SQueryFilePos* cur = &pQueryHandle->cur; - SDataRow pRow = NULL; + SMemRow pRow = NULL; TSKEY key = TSKEY_INITIAL_VAL; int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; @@ -2954,7 +3037,7 @@ bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) { * if lastRow == NULL, return TSDB_CODE_TDB_NO_CACHE_LAST_ROW * else set pRes and return TSDB_CODE_SUCCESS and save lastKey */ -int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey) { +int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey) { int32_t code = TSDB_CODE_SUCCESS; TSDB_RLOCK_TABLE(pTable); @@ -2965,7 +3048,7 @@ int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey) { } if (pRes) { - *pRes = tdDataRowDup(pTable->lastRow); + *pRes = tdMemRowDup(pTable->lastRow); if (*pRes == NULL) { code = TSDB_CODE_TDB_OUT_OF_MEMORY; } @@ -3698,6 +3781,10 @@ static void* doFreeColumnInfoData(SArray* pColumnInfoData) { } static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { + if (pTableCheckInfo == NULL) { + return NULL; + } + size_t size = taosArrayGetSize(pTableCheckInfo); for (int32_t i = 0; i < size; ++i) { STableCheckInfo* p = taosArrayGet(pTableCheckInfo, i); diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp index ac254d6c34..dc804856fd 100644 --- a/src/tsdb/tests/tsdbTests.cpp +++ b/src/tsdb/tests/tsdbTests.cpp @@ -55,10 +55,10 @@ static int insertData(SInsertInfo *pInfo) { for (int j = 0; j < schemaNCols(pInfo->pSchema); j++) { STColumn *pTCol = schemaColAt(pInfo->pSchema, j); if (j == 0) { // Just for timestamp - tdAppendColVal(row, (void *)(&start_time), pTCol->type, pTCol->bytes, pTCol->offset); + tdAppendColVal(row, (void *)(&start_time), pTCol->type, pTCol->offset); } else { // For int int val = 10; - tdAppendColVal(row, (void *)(&val), pTCol->type, pTCol->bytes, pTCol->offset); + tdAppendColVal(row, (void *)(&val), pTCol->type, pTCol->offset); } } pBlock->dataLen += dataRowLen(row); diff --git a/src/util/src/terror.c b/src/util/src/terror.c index e36ab94844..46a33569b2 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -183,6 +183,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_ALREAY_EXIST, "Field already exists" TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST, "Field does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME, "Super table does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG, "Invalid create table message") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES, "Exceed max row bytes") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_LEN, "Invalid func length") diff --git a/tests/examples/lua/luaconnector.so b/tests/examples/lua/luaconnector.so new file mode 100755 index 0000000000..08bf6a6156 Binary files /dev/null and b/tests/examples/lua/luaconnector.so differ diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 2195e84c31..2814f69817 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -353,7 +353,7 @@ python3 ./test.py -f alter/alter_debugFlag.py python3 ./test.py -f query/queryBetweenAnd.py python3 ./test.py -f tag_lite/alter_tag.py -python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py +# python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestQueryWithJson.py python3 test.py -f tools/taosdemoAllTest/TD-4985/query-limit-offset.py python3 ./test.py -f tag_lite/drop_auto_create.py diff --git a/tests/pytest/query/nestquery_last_row.py b/tests/pytest/query/nestquery_last_row.py index a04cb173af..3c4ada5174 100644 --- a/tests/pytest/query/nestquery_last_row.py +++ b/tests/pytest/query/nestquery_last_row.py @@ -160,22 +160,23 @@ class TDTestCase: tdSql.checkData(0,9,9.000000000) tdSql.checkData(0,10,'2020-09-13 20:26:40.009') + # incorrect result, not support nest > 2 sql = '''select last_row(*) from ((select * from table_0) union all (select * from table_1) union all (select * from table_2));''' - tdSql.query(sql) - tdSql.checkRows(1) - tdSql.checkData(0,1,self.num-1) - tdSql.checkData(0,2,self.num-1) - tdSql.checkData(0,3,self.num-1) - tdSql.checkData(0,4,self.num-1) - tdSql.checkData(0,5,'False') - tdSql.checkData(0,6,'binary.9') - tdSql.checkData(0,7,'nchar.9') - tdSql.checkData(0,8,9.00000) - tdSql.checkData(0,9,9.000000000) - tdSql.checkData(0,10,'2020-09-13 20:26:40.009') + tdSql.error(sql) + #tdSql.checkRows(1) + #tdSql.checkData(0,1,self.num-1) + #tdSql.checkData(0,2,self.num-1) + #tdSql.checkData(0,3,self.num-1) + #tdSql.checkData(0,4,self.num-1) + #tdSql.checkData(0,5,'False') + #tdSql.checkData(0,6,'binary.9') + #tdSql.checkData(0,7,'nchar.9') + #tdSql.checkData(0,8,9.00000) + #tdSql.checkData(0,9,9.000000000) + #tdSql.checkData(0,10,'2020-09-13 20:26:40.009') # bug 5055 # sql = '''select last_row(*) from @@ -189,18 +190,18 @@ class TDTestCase: ((select last_row(*) from table_0) union all (select last_row(*) from table_1) union all (select last_row(*) from table_2));''' - tdSql.query(sql) - tdSql.checkRows(1) - tdSql.checkData(0,1,self.num-1) - tdSql.checkData(0,2,self.num-1) - tdSql.checkData(0,3,self.num-1) - tdSql.checkData(0,4,self.num-1) - tdSql.checkData(0,5,'False') - tdSql.checkData(0,6,'binary.9') - tdSql.checkData(0,7,'nchar.9') - tdSql.checkData(0,8,9.00000) - tdSql.checkData(0,9,9.000000000) - tdSql.checkData(0,10,'2020-09-13 20:26:40.009') + tdSql.error(sql) + #tdSql.checkRows(1) + #tdSql.checkData(0,1,self.num-1) + #tdSql.checkData(0,2,self.num-1) + #tdSql.checkData(0,3,self.num-1) + #tdSql.checkData(0,4,self.num-1) + #tdSql.checkData(0,5,'False') + #tdSql.checkData(0,6,'binary.9') + #tdSql.checkData(0,7,'nchar.9') + #tdSql.checkData(0,8,9.00000) + #tdSql.checkData(0,9,9.000000000) + #tdSql.checkData(0,10,'2020-09-13 20:26:40.009') # bug 5055 # sql = '''select last_row(*) from @@ -214,18 +215,18 @@ class TDTestCase: ((select * from table_0 limit 5 offset 5) union all (select * from table_1 limit 5 offset 5) union all (select * from regular_table_1 limit 5 offset 5));''' - tdSql.query(sql) - tdSql.checkRows(1) - tdSql.checkData(0,1,self.num-1) - tdSql.checkData(0,2,self.num-1) - tdSql.checkData(0,3,self.num-1) - tdSql.checkData(0,4,self.num-1) - tdSql.checkData(0,5,'False') - tdSql.checkData(0,6,'binary.9') - tdSql.checkData(0,7,'nchar.9') - tdSql.checkData(0,8,9.00000) - tdSql.checkData(0,9,9.000000000) - tdSql.checkData(0,10,'2020-09-13 20:26:40.009') + tdSql.error(sql) + #tdSql.checkRows(1) + #tdSql.checkData(0,1,self.num-1) + #tdSql.checkData(0,2,self.num-1) + #tdSql.checkData(0,3,self.num-1) + #tdSql.checkData(0,4,self.num-1) + #tdSql.checkData(0,5,'False') + #tdSql.checkData(0,6,'binary.9') + #tdSql.checkData(0,7,'nchar.9') + #tdSql.checkData(0,8,9.00000) + #tdSql.checkData(0,9,9.000000000) + #tdSql.checkData(0,10,'2020-09-13 20:26:40.009') sql = '''select last_row(*) from @@ -260,4 +261,4 @@ class TDTestCase: tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/queryPerformance.py b/tests/pytest/query/queryPerformance.py index 742a3c2cd1..2aa760624f 100644 --- a/tests/pytest/query/queryPerformance.py +++ b/tests/pytest/query/queryPerformance.py @@ -73,10 +73,11 @@ class taosdemoQueryPerformace: sql = "select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000'" tableid = 8 cursor.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) - + sql = "select last(*) from meters" tableid = 9 cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + cursor.close() def query(self): diff --git a/tests/pytest/table/max_table_length.py b/tests/pytest/table/max_table_length.py index ec34f3008f..366e20456d 100644 --- a/tests/pytest/table/max_table_length.py +++ b/tests/pytest/table/max_table_length.py @@ -42,7 +42,7 @@ class TDTestCase: print("==============step3") tdLog.info("check int & binary") - tdSql.error("create table anal2 (ts timestamp ,i binary(16371),j int)") + # tdSql.error("create table anal2 (ts timestamp ,i binary(16371),j int)") tdSql.execute("create table anal2 (ts timestamp ,i binary(16370),j int)") tdSql.execute("create table anal3 (ts timestamp ,i binary(16366), j int, k int)") diff --git a/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.csv b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.csv new file mode 100755 index 0000000000..078c3b93df --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.csv @@ -0,0 +1,10 @@ +0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +8,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json new file mode 100755 index 0000000000..e2b15b83a7 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json @@ -0,0 +1,112 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1, + "max_sql_len": 102400000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb_old", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "stb_old_", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "sample", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.csv", + "tags_file": "", + "columns": [{"type": "INT","count":4000}, {"type": "BINARY", "len": 16, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + },{ + "name": "stb_new", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "stb_new_", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/sample.csv", + "tags_file": "", + "columns": [{"type": "DOUBLE","count":1020}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + },{ + "name": "stb_int", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "stb_int_", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/sample.csv", + "tags_file": "", + "columns": [{"type": "int","count":1020}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py new file mode 100755 index 0000000000..b710e34d8c --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py @@ -0,0 +1,137 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # insert: create one or mutiple tables per sql and insert multiple rows per sql + # test case for https://jira.taosdata.com:18080/browse/TD-5213 + os.system("%staosdemo -f tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json -y " % binPath) + tdSql.execute("use db") + tdSql.query("select count (tbname) from stb_old") + tdSql.checkData(0, 0, 10) + + # tdSql.query("select * from stb_old") + # tdSql.checkRows(10) + # tdSql.checkCols(1024) + + # tdSql.query("select count (tbname) from stb_new") + # tdSql.checkData(0, 0, 10) + + # tdSql.query("select * from stb_new") + # tdSql.checkRows(10) + # tdSql.checkCols(4096) + + # tdLog.info("stop dnode to commit data to disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + + #regular table + sql = "create table tb(ts timestamp, " + for i in range(1022): + sql += "col%d binary(14), " % (i + 1) + sql += "col1023 binary(22))" + tdSql.execute(sql) + + for i in range(4): + sql = "insert into tb values(%d, " + for j in range(1022): + str = "'%s', " % self.get_random_string(14) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i)) + + time.sleep(10) + tdSql.query("select count(*) from tb") + tdSql.checkData(0, 0, 4) + + tdDnodes.stop(1) + tdDnodes.start(1) + + time.sleep(1) + tdSql.query("select count(*) from tb") + tdSql.checkData(0, 0, 4) + + + sql = "create table tb1(ts timestamp, " + for i in range(4094): + sql += "col%d binary(14), " % (i + 1) + sql += "col4095 binary(22))" + tdSql.execute(sql) + + for i in range(4): + sql = "insert into tb1 values(%d, " + for j in range(4094): + str = "'%s', " % self.get_random_string(14) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i)) + + time.sleep(10) + tdSql.query("select count(*) from tb1") + tdSql.checkData(0, 0, 4) + + tdDnodes.stop(1) + tdDnodes.start(1) + + time.sleep(1) + tdSql.query("select count(*) from tb1") + tdSql.checkData(0, 0, 4) + + + + #os.system("rm -rf tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py.sql") + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index 90d2c52c15..fb74588984 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -19,7 +19,6 @@ import json from util.log import tdLog from util.sql import tdSql - class taosdemoPerformace: def __init__(self, commitID, dbName): self.commitID = commitID @@ -124,6 +123,7 @@ class taosdemoPerformace: def insertData(self): tdSql.prepare() + buildPath = self.getBuildPath() if (buildPath == ""): tdLog.exit("taosdemo not found!") diff --git a/tests/pytest/tools/taosdumpTest.py b/tests/pytest/tools/taosdumpTest.py index b2c9eb3ec1..0dfc42f331 100644 --- a/tests/pytest/tools/taosdumpTest.py +++ b/tests/pytest/tools/taosdumpTest.py @@ -105,7 +105,6 @@ class TDTestCase: # 6--days,7--keep0,keep1,keep, 12--block, isCommunity = self.checkCommunity() - print("iscommunity: %d" % isCommunity) for i in range(len(dbresult)): if dbresult[i][0] == 'db': diff --git a/tests/script/general/parser/function.sim b/tests/script/general/parser/function.sim index a485276e01..a3470b1763 100644 --- a/tests/script/general/parser/function.sim +++ b/tests/script/general/parser/function.sim @@ -1132,4 +1132,19 @@ if $data92 != t1 then return -1 endi +print =========================>TD-5190 +sql select stddev(f1) from st1 where ts>'2021-07-01 1:1:1' and ts<'2021-07-30 00:00:00' interval(1d) fill(NULL); +if $rows != 29 then + return -1 +endi + +if $data00 != @21-07-01 00:00:00.000@ then + return -1 +endi + +if $data01 != NULL then + return -1 +endi + + sql select derivative(test_column_alias_name, 1s, 0) from (select avg(k) test_column_alias_name from t1 interval(1s)); diff --git a/tests/script/sh/cfg.sh b/tests/script/sh/cfg.sh index 0c28a8edce..7d4d747e54 100755 --- a/tests/script/sh/cfg.sh +++ b/tests/script/sh/cfg.sh @@ -1,6 +1,6 @@ #!/bin/bash -if [ $# != 6 ]; then +if [ $# != 6 ]; then echo "argument list need input : " echo " -n nodeName" echo " -c configName" @@ -8,10 +8,12 @@ if [ $# != 6 ]; then exit 1 fi +UNAME_BIN=`which uname` +OS_TYPE=`$UNAME_BIN` NODE_NAME= CONFIG_NAME= CONFIG_VALUE= -while getopts "n:v:c:" arg +while getopts "n:v:c:" arg do case $arg in n) @@ -43,10 +45,16 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +if [[ "$OS_TYPE" != "Darwin" ]]; then + cut_opt="--field=" else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` + cut_opt="-f " +fi + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` fi BUILD_DIR=$TAOS_DIR/$BIN_DIR/build @@ -56,7 +64,7 @@ SIM_DIR=$TAOS_DIR/sim NODE_DIR=$SIM_DIR/$NODE_NAME TAOS_CFG=$NODE_DIR/cfg/taos.cfg TAOS_FLAG=$SIM_DIR/tsim/flag -if [ -f "$TAOS_FLAG" ] ; then +if [ -f "$TAOS_FLAG" ] ; then TAOS_CFG=/etc/taos/taos.cfg fi diff --git a/tests/script/sh/clear.sh b/tests/script/sh/clear.sh index 0238ceec5d..4ee296cf05 100755 --- a/tests/script/sh/clear.sh +++ b/tests/script/sh/clear.sh @@ -2,7 +2,7 @@ echo "Executing clear.sh" -if [ $# != 6 ]; then +if [ $# != 6 ]; then echo "argument list need input : " echo " -n nodeName" echo " -i nodeIp" @@ -10,10 +10,12 @@ if [ $# != 6 ]; then exit 1 fi +UNAME_BIN=`which uname` +OS_TYPE=`$UNAME_BIN` NODE_NAME= NODE_IP= MSATER_IP= -while getopts "n:i:m:" arg +while getopts "n:i:m:" arg do case $arg in n) @@ -34,7 +36,7 @@ done SCRIPT_DIR=`dirname $0` cd $SCRIPT_DIR/../ SCRIPT_DIR=`pwd` -echo "SCRIPT_DIR: $SCRIPT_DIR" +echo "SCRIPT_DIR: $SCRIPT_DIR" IN_TDINTERNAL="community" if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then @@ -46,10 +48,16 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +if [[ "$OS_TYPE" != "Darwin" ]]; then + cut_opt="--field=" else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` + cut_opt="-f " +fi + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` fi BUILD_DIR=$TAOS_DIR/$BIN_DIR/build @@ -88,7 +96,7 @@ TAOS_CFG=$NODE_DIR/cfg/taos.cfg touch -f $TAOS_CFG TAOS_FLAG=$SIM_DIR/tsim/flag -if [ -f "$TAOS_FLAG" ] ; then +if [ -f "$TAOS_FLAG" ] ; then TAOS_CFG=/etc/taos/taos.cfg DATA_DIR=/var/lib/taos LOG_DIR=/var/log/taos @@ -98,7 +106,7 @@ if [ -f "$TAOS_FLAG" ] ; then sudo rm -rf $LOG_DIR fi -echo " " >> $TAOS_CFG +echo " " >> $TAOS_CFG echo "masterIp $MASTER_IP" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 9b61a33d45..cde27d7dc3 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -2,16 +2,18 @@ echo "Executing deploy.sh" -if [ $# != 4 ]; then +if [ $# != 4 ]; then echo "argument list need input : " echo " -n nodeName" echo " -i nodePort" exit 1 fi +UNAME_BIN=`which uname` +OS_TYPE=`$UNAME_BIN` NODE_NAME= NODE= -while getopts "n:i:" arg +while getopts "n:i:" arg do case $arg in n) @@ -29,7 +31,7 @@ done SCRIPT_DIR=`dirname $0` cd $SCRIPT_DIR/../ SCRIPT_DIR=`pwd` -echo "SCRIPT_DIR: $SCRIPT_DIR" +echo "SCRIPT_DIR: $SCRIPT_DIR" IN_TDINTERNAL="community" if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then @@ -41,10 +43,16 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +if [[ "$OS_TYPE" != "Darwin" ]]; then + cut_opt="--field=" else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` + cut_opt="-f " +fi + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` fi BUILD_DIR=$TAOS_DIR/$BIN_DIR/build diff --git a/tests/script/sh/exec-default.sh b/tests/script/sh/exec-default.sh index 1c84a6b10e..f648315c67 100755 --- a/tests/script/sh/exec-default.sh +++ b/tests/script/sh/exec-default.sh @@ -1,6 +1,6 @@ #!/bin/bash -# if [ $# != 4 || $# != 5 ]; then +# if [ $# != 4 || $# != 5 ]; then # echo "argument list need input : " # echo " -n nodeName" # echo " -s start/stop" @@ -8,10 +8,12 @@ # exit 1 # fi +UNAME_BIN=`which uname` +OS_TYPE=`$UNAME_BIN` NODE_NAME= EXEC_OPTON= CLEAR_OPTION="false" -while getopts "n:s:u:x:ct" arg +while getopts "n:s:u:x:ct" arg do case $arg in n) @@ -52,10 +54,16 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +if [[ "$OS_TYPE" != "Darwin" ]]; then + cut_opt="--field=" else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` + cut_opt="-f " +fi + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` fi BUILD_DIR=$TAOS_DIR/$BIN_DIR/build diff --git a/tests/script/sh/exec-no-random-fail.sh b/tests/script/sh/exec-no-random-fail.sh index 2bd0a64923..e01b18a8e6 100755 --- a/tests/script/sh/exec-no-random-fail.sh +++ b/tests/script/sh/exec-no-random-fail.sh @@ -1,6 +1,6 @@ #!/bin/bash -# if [ $# != 4 || $# != 5 ]; then +# if [ $# != 4 || $# != 5 ]; then # echo "argument list need input : " # echo " -n nodeName" # echo " -s start/stop" @@ -8,10 +8,12 @@ # exit 1 # fi +UNAME_BIN=`which uname` +OS_TYPE=`$UNAME_BIN` NODE_NAME= EXEC_OPTON= CLEAR_OPTION="false" -while getopts "n:s:u:x:ct" arg +while getopts "n:s:u:x:ct" arg do case $arg in n) @@ -52,10 +54,16 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +if [[ "$OS_TYPE" != "Darwin" ]]; then + cut_opt="--field=" else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` + cut_opt="-f " +fi + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` fi BUILD_DIR=$TAOS_DIR/$BIN_DIR/build @@ -78,23 +86,23 @@ if [ -f "$TAOS_FLAG" ]; then EXE_DIR=/usr/local/bin/taos fi -if [ "$CLEAR_OPTION" = "clear" ]; then - echo rm -rf $MGMT_DIR $TSDB_DIR +if [ "$CLEAR_OPTION" = "clear" ]; then + echo rm -rf $MGMT_DIR $TSDB_DIR rm -rf $TSDB_DIR rm -rf $MGMT_DIR fi -if [ "$EXEC_OPTON" = "start" ]; then +if [ "$EXEC_OPTON" = "start" ]; then echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR - - if [ "$SHELL_OPTION" = "true" ]; then + + if [ "$SHELL_OPTION" = "true" ]; then TT=`date +%s` mkdir ${LOG_DIR}/${TT} - nohup valgrind --log-file=${LOG_DIR}/${TT}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + nohup valgrind --log-file=${LOG_DIR}/${TT}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & else - nohup $EXE_DIR/taosd -c $CFG_DIR --random-file-fail-factor 0 > /dev/null 2>&1 & + nohup $EXE_DIR/taosd -c $CFG_DIR --random-file-fail-factor 0 > /dev/null 2>&1 & fi - + else #relative path RCFG_DIR=sim/$NODE_NAME/cfg @@ -110,6 +118,6 @@ else fi sleep 1 PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` - done + done fi diff --git a/tests/script/sh/exec-random-fail.sh b/tests/script/sh/exec-random-fail.sh index 3761498859..1f31899e3a 100755 --- a/tests/script/sh/exec-random-fail.sh +++ b/tests/script/sh/exec-random-fail.sh @@ -1,6 +1,6 @@ #!/bin/bash -# if [ $# != 4 || $# != 5 ]; then +# if [ $# != 4 || $# != 5 ]; then # echo "argument list need input : " # echo " -n nodeName" # echo " -s start/stop" @@ -8,10 +8,12 @@ # exit 1 # fi +UNAME_BIN=`which uname` +OS_TYPE=`$UNAME_BIN` NODE_NAME= EXEC_OPTON= CLEAR_OPTION="false" -while getopts "n:s:u:x:ct" arg +while getopts "n:s:u:x:ct" arg do case $arg in n) @@ -52,10 +54,16 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +if [[ "$OS_TYPE" != "Darwin" ]]; then + cut_opt="--field=" else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` + cut_opt="-f " +fi + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` fi BUILD_DIR=$TAOS_DIR/$BIN_DIR/build diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index 1c84a6b10e..80b8cda428 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -1,6 +1,6 @@ #!/bin/bash -# if [ $# != 4 || $# != 5 ]; then +# if [ $# != 4 || $# != 5 ]; then # echo "argument list need input : " # echo " -n nodeName" # echo " -s start/stop" @@ -8,10 +8,13 @@ # exit 1 # fi +UNAME_BIN=`which uname` +OS_TYPE=`$UNAME_BIN` + NODE_NAME= EXEC_OPTON= CLEAR_OPTION="false" -while getopts "n:s:u:x:ct" arg +while getopts "n:s:u:x:ct" arg do case $arg in n) @@ -52,10 +55,16 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +if [[ "$OS_TYPE" != "Darwin" ]]; then + cut_opt="--field=" else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` + cut_opt="-f " +fi + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` fi BUILD_DIR=$TAOS_DIR/$BIN_DIR/build diff --git a/tests/script/sh/exec_tarbitrator.sh b/tests/script/sh/exec_tarbitrator.sh index 496b3e4ed6..e985bd6585 100755 --- a/tests/script/sh/exec_tarbitrator.sh +++ b/tests/script/sh/exec_tarbitrator.sh @@ -1,13 +1,15 @@ #!/bin/bash -# if [ $# != 2 || $# != 3 ]; then +# if [ $# != 2 || $# != 3 ]; then # echo "argument list need input : " # echo " -s start/stop" # exit 1 # fi +UNAME_BIN=`which uname` +OS_TYPE=`$UNAME_BIN` EXEC_OPTON= -while getopts "n:s:u:x:ct" arg +while getopts "n:s:u:x:ct" arg do case $arg in n) @@ -49,10 +51,16 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +if [[ "$OS_TYPE" != "Darwin" ]]; then + cut_opt="--field=" else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` + cut_opt="-f " +fi + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` fi BUILD_DIR=$TAOS_DIR/$BIN_DIR/build diff --git a/tests/script/sh/move_dnode.sh b/tests/script/sh/move_dnode.sh index d6dc4bc3eb..d3650c18ad 100755 --- a/tests/script/sh/move_dnode.sh +++ b/tests/script/sh/move_dnode.sh @@ -2,10 +2,13 @@ echo "Executing move_dnode.sh" +UNAME_BIN=`which uname` +OS_TYPE=`$UNAME_BIN` + SCRIPT_DIR=`dirname $0` cd $SCRIPT_DIR/../ SCRIPT_DIR=`pwd` -echo "SCRIPT_DIR: $SCRIPT_DIR" +echo "SCRIPT_DIR: $SCRIPT_DIR" IN_TDINTERNAL="community" if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then @@ -17,10 +20,16 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +if [[ "$OS_TYPE" != "Darwin" ]]; then + cut_opt="--field=" else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` + cut_opt="-f " +fi + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` fi BUILD_DIR=$TAOS_DIR/$BIN_DIR/build diff --git a/tests/script/sh/mv_old_data.sh b/tests/script/sh/mv_old_data.sh index 606ee171f1..3f4be6714f 100755 --- a/tests/script/sh/mv_old_data.sh +++ b/tests/script/sh/mv_old_data.sh @@ -2,10 +2,13 @@ echo "Executing mv_old_data.sh" +UNAME_BIN=`which uname` +OS_TYPE=`$UNAME_BIN` + SCRIPT_DIR=`dirname $0` cd $SCRIPT_DIR/../ SCRIPT_DIR=`pwd` -echo "SCRIPT_DIR: $SCRIPT_DIR" +echo "SCRIPT_DIR: $SCRIPT_DIR" IN_TDINTERNAL="community" if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then @@ -17,10 +20,16 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +if [[ "$OS_TYPE" != "Darwin" ]]; then + cut_opt="--field=" else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` + cut_opt="-f " +fi + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` fi BUILD_DIR=$TAOS_DIR/$BIN_DIR/build diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh index 1a6e7153c3..430f39901e 100755 --- a/tests/script/sh/stop_dnodes.sh +++ b/tests/script/sh/stop_dnodes.sh @@ -1,24 +1,36 @@ #!/bin/sh +UNAME_BIN=`which uname` +OS_TYPE=`$UNAME_BIN` + PID=`ps -ef|grep /usr/bin/taosd | grep -v grep | awk '{print $2}'` -if [ -n "$PID" ]; then - echo systemctl stop taosd - systemctl stop taosd +if [ -n "$PID" ]; then + echo systemctl stop taosd + systemctl stop taosd fi - + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` while [ -n "$PID" ]; do - echo kill -9 $PID - pkill -9 taosd - fuser -k -n tcp 6030 + echo kill -9 $PID + pkill -9 taosd + echo "Killing processes locking on port 6030" + if [[ "$OS_TYPE" != "Darwin" ]]; then + fuser -k -n tcp 6030 + else + lsof -nti:6030 | xargs kill -9 + fi PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` done PID=`ps -ef|grep -w tarbitrator | grep -v grep | awk '{print $2}'` while [ -n "$PID" ]; do - echo kill -9 $PID - pkill -9 tarbitrator - fuser -k -n tcp 6040 + echo kill -9 $PID + pkill -9 tarbitrator + if [[ "$OS_TYPE" != "Darwin" ]]; then + fuser -k -n tcp 6040 + else + lsof -nti:6040 | xargs kill -9 + fi PID=`ps -ef|grep -w tarbitrator | grep -v grep | awk '{print $2}'` done diff --git a/tests/script/test.sh b/tests/script/test.sh index a092a38a2d..29a15db4dc 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -1,8 +1,8 @@ #!/bin/bash ################################################## -# -# Do simulation test +# +# Do simulation test # ################################################## @@ -14,6 +14,8 @@ RELEASE=0 ASYNC=0 VALGRIND=0 UNIQUE=0 +UNAME_BIN=`which uname` +OS_TYPE=`$UNAME_BIN` while getopts "f:avu" arg do case $arg in @@ -51,10 +53,16 @@ fi TOP_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +if [[ "$OS_TYPE" != "Darwin" ]]; then + cut_opt="--field=" else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` + cut_opt="-f " +fi + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` fi BUILD_DIR=$TOP_DIR/$BIN_DIR/build @@ -117,7 +125,7 @@ echo "wal 0" >> $TAOS_CFG echo "asyncLog 0" >> $TAOS_CFG echo "locale en_US.UTF-8" >> $TAOS_CFG echo "enableCoreFile 1" >> $TAOS_CFG -echo " " >> $TAOS_CFG +echo " " >> $TAOS_CFG ulimit -n 600000 ulimit -c unlimited