From 8412703d8560c65c9dd3214f7025ca08d8a4d4e1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 Aug 2021 19:19:03 +0800 Subject: [PATCH 01/75] [td-225] --- src/query/inc/qExecutor.h | 10 ++- src/query/src/qExecutor.c | 139 +++++++++++++++++++++++++++++++++++--- src/util/src/tcompare.c | 4 -- 3 files changed, 140 insertions(+), 13 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index ce70a9ba4a..c9675c3844 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -331,6 +331,7 @@ enum OPERATOR_TYPE_E { OP_Distinct = 20, OP_Join = 21, OP_StateWindow = 22, + OP_Order = 23, }; typedef struct SOperatorInfo { @@ -506,7 +507,7 @@ typedef struct SStateWindowOperatorInfo { int32_t start; char* prevData; // previous data bool reptScan; -} SStateWindowOperatorInfo ; +} SStateWindowOperatorInfo; typedef struct SDistinctOperatorInfo { SHashObj *pSet; @@ -539,6 +540,13 @@ typedef struct SMultiwayMergeInfo { SArray *udfInfo; } SMultiwayMergeInfo; +// todo support the disk-based sort +typedef struct SOrderOperatorInfo { + int32_t colIndex; + int32_t order; + SSDataBlock *pDataBlock; +} SOrderOperatorInfo; + void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3f6df2ec07..fb9f5d93d4 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -16,7 +16,6 @@ #include "qFill.h" #include "taosmsg.h" #include "tglobal.h" -#include "talgo.h" #include "exception.h" #include "hash.h" @@ -242,8 +241,7 @@ static void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx static void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); static void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); static void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, - SQLFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, - int32_t groupIndex); + SQLFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId); SArray* getOrderCheckColumns(SQueryAttr* pQuery); @@ -886,8 +884,6 @@ void doInvokeUdf(SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t break; } } - - return; } static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, @@ -5218,6 +5214,35 @@ SOperatorInfo *createMultiwaySortOperatorInfo(SQueryRuntimeEnv *pRuntimeEnv, SEx return pOperator; } +SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv *pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows) { + SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo)); + + { + SSDataBlock* pDataBlock = calloc(1, sizeof(SSDataBlock)); + pDataBlock->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); + for(int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData col = {0}; + col.info.bytes = pExpr->base.resBytes; + col.info.colId = pExpr->base.resColId; + col.info.type = pExpr->base.resType; + taosArrayPush(pDataBlock->pDataBlock, &col); + } + + pDataBlock->info.numOfCols = numOfOutput; + } + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "Order"; + pOperator->operatorType = OP_Order; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->exec = doSort; + pOperator->cleanup = NULL; + + return pOperator; +} + static int32_t getTableScanOrder(STableScanInfo* pTableScanInfo) { return pTableScanInfo->order; } @@ -5642,7 +5667,6 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { return pIntervalInfo->pRes; } - static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; STableQueryInfo* item = pRuntimeEnv->current; @@ -5775,6 +5799,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { return pBInfo->pRes->info.rows == 0? NULL:pBInfo->pRes; } + static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) { SOperatorInfo* pOperator = (SOperatorInfo*) param; if (pOperator->status == OP_EXEC_DONE) { @@ -5986,6 +6011,106 @@ static SSDataBlock* doFill(void* param, bool* newgroup) { } } +static int32_t doMergeSDatablock(SSDataBlock* pDest, SSDataBlock* pSrc) { + assert(pSrc != NULL && pDest != NULL && pDest->info.numOfCols == pSrc->info.numOfCols); + + int32_t numOfCols = pSrc->info.numOfCols; + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock->pData, i); + SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock->pData, i); + + int32_t newSize = (pDest->info.rows + pSrc->info.rows) * pCol2->info.bytes; + char* tmp = realloc(pCol2->pData, newSize); + if (tmp != NULL) { + pCol2->pData = tmp; + int32_t offset = pCol2->info.bytes * pDest->info.rows; + memcpy(pCol2->pData + offset, pCol1->pData, pSrc->info.rows * pCol2->info.bytes); + } else { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* doSort(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SOrderOperatorInfo* pInfo = pOperator->info; +// SSDataBlock* pRes = pInfo->pRes; + +// pRes->info.rows = 0; + SSDataBlock* pBlock = NULL; + while(1) { + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + + // start to flush data into disk and try do multiway merge sort + if (pBlock == NULL) { +// setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); +// pOperator->status = OP_EXEC_DONE; + break; + } + + int32_t code = doMergeSDatablock(pInfo->pDataBlock, pBlock); + if (code != TSDB_CODE_SUCCESS) { +// return code; + } + + /*SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex); + + int16_t bytes = pColInfoData->info.bytes; + int16_t type = pColInfoData->info.type; + + // ensure the output buffer size + SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, 0); + if (pRes->info.rows + pBlock->info.rows > pInfo->outputCapacity) { + int32_t newSize = pRes->info.rows + pBlock->info.rows; + char* tmp = realloc(pResultColInfoData->pData, newSize * bytes); + if (tmp == NULL) { + return NULL; + } else { + pResultColInfoData->pData = tmp; + pInfo->outputCapacity = newSize; + } + } + + for(int32_t i = 0; i < pBlock->info.rows; ++i) { + char* val = ((char*)pColInfoData->pData) + bytes * i; + if (isNull(val, type)) { + continue; + } + + size_t keyLen = 0; + if (IS_VAR_DATA_TYPE(pOperator->pExpr->base.colType)) { + tstr* var = (tstr*)(val); + keyLen = varDataLen(var); + } else { + keyLen = bytes; + } + + int dummy; + void* res = taosHashGet(pInfo->pSet, val, keyLen); + if (res == NULL) { + taosHashPut(pInfo->pSet, val, keyLen, &dummy, sizeof(dummy)); + char* start = pResultColInfoData->pData + bytes * pInfo->pRes->info.rows; + memcpy(start, val, bytes); + pRes->info.rows += 1; + } + } + + if (pRes->info.rows >= pInfo->threshold) { + break; + }*/ + } + + return (pInfo->pDataBlock->info.rows > 0)? pInfo->pDataBlock:NULL; +} + // todo set the attribute of query scan count static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr) { for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { @@ -6607,11 +6732,9 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) { return NULL; } - SDistinctOperatorInfo* pInfo = pOperator->info; SSDataBlock* pRes = pInfo->pRes; - pRes->info.rows = 0; SSDataBlock* pBlock = NULL; while(1) { diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index e953f4c464..55ba14f84f 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -286,10 +286,6 @@ int32_t taosArrayCompareString(const void* a, const void* b) { return compareLenPrefixedStr(x, y); } -//static int32_t compareFindStrInArray(const void* pLeft, const void* pRight) { -// const SArray* arr = (const SArray*) pRight; -// return taosArraySearchString(arr, pLeft, taosArrayCompareString, TD_EQ) == NULL ? 0 : 1; -//} static int32_t compareFindItemInSet(const void *pLeft, const void* pRight) { return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; } From 8ad04a513c67d1437f1038fcfbb954fed7a68ef7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 4 Aug 2021 19:21:34 +0800 Subject: [PATCH 02/75] [td-5796]: fix the invalid error message. --- src/client/src/tscServer.c | 6 ++++-- src/client/src/tscSubquery.c | 5 +++-- src/client/src/tscUtil.c | 28 +++++++++++++++------------- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 57aefac852..7120d780f9 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2980,11 +2980,13 @@ int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid); } + // remove stored tableMeta info in hash table tscRemoveTableMetaBuf(pTableMetaInfo, pSql->self); + tscResetSqlCmd(pCmd, true); - pCmd->pTableMetaMap = tscCleanupTableMetaMap(pCmd->pTableMetaMap); - pCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); +// pCmd->pTableMetaMap = tscCleanupTableMetaMap(pCmd->pTableMetaMap); +// pCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); SArray* pNameList = taosArrayInit(1, POINTER_BYTES); SArray* vgroupList = taosArrayInit(1, POINTER_BYTES); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 8c0d642ca6..3816e6d619 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2721,9 +2721,10 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO SSqlCmd* pParentCmd = &pParentSql->cmd; STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pParentCmd, 0); tscRemoveTableMetaBuf(pTableMetaInfo, pParentSql->self); + tscResetSqlCmd(pParentCmd, true); - pParentCmd->pTableMetaMap = tscCleanupTableMetaMap(pParentCmd->pTableMetaMap); - pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); +// pParentCmd->pTableMetaMap = tscCleanupTableMetaMap(pParentCmd->pTableMetaMap); +// pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); pParentSql->res.code = TSDB_CODE_SUCCESS; pParentSql->retry++; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 694d2b2af6..849cbb3aa8 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1374,18 +1374,19 @@ void tscResetSqlCmd(SSqlCmd* pCmd, bool clearCachedMeta) { pCmd->insertParam.tagData.dataLen = 0; tscFreeQueryInfo(pCmd, clearCachedMeta); + pCmd->pTableMetaMap = tscCleanupTableMetaMap(pCmd->pTableMetaMap); - if (pCmd->pTableMetaMap != NULL) { - STableMetaVgroupInfo* p = taosHashIterate(pCmd->pTableMetaMap, NULL); - while (p) { - taosArrayDestroy(p->vgroupIdList); - tfree(p->pTableMeta); - p = taosHashIterate(pCmd->pTableMetaMap, p); - } - - taosHashCleanup(pCmd->pTableMetaMap); - pCmd->pTableMetaMap = NULL; - } +// if (pCmd->pTableMetaMap != NULL) { +// STableMetaVgroupInfo* p = taosHashIterate(pCmd->pTableMetaMap, NULL); +// while (p) { +// taosArrayDestroy(p->vgroupIdList); +// tfree(p->pTableMeta); +// p = taosHashIterate(pCmd->pTableMetaMap, p); +// } +// +// taosHashCleanup(pCmd->pTableMetaMap); +// pCmd->pTableMetaMap = NULL; +// } } void* tscCleanupTableMetaMap(SHashObj* pTableMetaMap) { @@ -3845,9 +3846,10 @@ static void tscSubqueryCompleteCallback(void* param, TAOS_RES* tres, int code) { SSqlCmd* pParentCmd = &pParentSql->cmd; STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pParentCmd, 0); tscRemoveTableMetaBuf(pTableMetaInfo, pParentSql->self); + tscResetSqlCmd(pParentCmd, true); - pParentCmd->pTableMetaMap = tscCleanupTableMetaMap(pParentCmd->pTableMetaMap); - pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); +// pParentCmd->pTableMetaMap = tscCleanupTableMetaMap(pParentCmd->pTableMetaMap); +// pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); pParentSql->res.code = TSDB_CODE_SUCCESS; pParentSql->retry++; From 7466d787ce07085199cea07a14923a16a0cfc4d8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 5 Aug 2021 11:05:22 +0800 Subject: [PATCH 03/75] [td-5573]: support order clause in outer query. --- src/query/inc/qExtbuffer.h | 2 + src/query/src/qExecutor.c | 171 ++++++++++++++-------------------- src/query/src/qExtbuffer.c | 57 +++++++++++- src/query/src/qPercentile.c | 2 +- src/query/tests/cSortTest.cpp | 124 ++++++++++++++++++++++++ src/query/tests/unitTest.cpp | 2 - src/util/inc/tcompare.h | 2 +- src/util/src/tcompare.c | 95 ++++++++++++++----- src/util/src/tskiplist.c | 2 +- 9 files changed, 326 insertions(+), 131 deletions(-) create mode 100644 src/query/tests/cSortTest.cpp diff --git a/src/query/inc/qExtbuffer.h b/src/query/inc/qExtbuffer.h index cf0e8ce31a..b5ea9932b9 100644 --- a/src/query/inc/qExtbuffer.h +++ b/src/query/inc/qExtbuffer.h @@ -227,6 +227,8 @@ typedef int (*__col_compar_fn_t)(tOrderDescriptor *, int32_t numOfRows, int32_t void tColDataQSort(tOrderDescriptor *, int32_t numOfRows, int32_t start, int32_t end, char *data, int32_t orderType); +void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t index, __compar_fn_t compareFn); + int32_t compare_sa(tOrderDescriptor *, int32_t numOfRows, int32_t idx1, int32_t idx2, char *data); int32_t compare_sd(tOrderDescriptor *, int32_t numOfRows, int32_t idx1, int32_t idx2, char *data); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index fb9f5d93d4..f836d83730 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5214,7 +5214,76 @@ SOperatorInfo *createMultiwaySortOperatorInfo(SQueryRuntimeEnv *pRuntimeEnv, SEx return pOperator; } -SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv *pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows) { +static int32_t doMergeSDatablock(SSDataBlock* pDest, SSDataBlock* pSrc) { + assert(pSrc != NULL && pDest != NULL && pDest->info.numOfCols == pSrc->info.numOfCols); + + int32_t numOfCols = pSrc->info.numOfCols; + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock->pData, i); + SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock->pData, i); + + int32_t newSize = (pDest->info.rows + pSrc->info.rows) * pCol2->info.bytes; + char* tmp = realloc(pCol2->pData, newSize); + if (tmp != NULL) { + pCol2->pData = tmp; + int32_t offset = pCol2->info.bytes * pDest->info.rows; + memcpy(pCol2->pData + offset, pCol1->pData, pSrc->info.rows * pCol2->info.bytes); + } else { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* doSort(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SOrderOperatorInfo* pInfo = pOperator->info; + + SSDataBlock* pBlock = NULL; + while(1) { + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + + // start to flush data into disk and try do multiway merge sort + if (pBlock == NULL) { + setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + pOperator->status = OP_EXEC_DONE; + break; + } + + int32_t code = doMergeSDatablock(pInfo->pDataBlock, pBlock); + if (code != TSDB_CODE_SUCCESS) { + // todo handle error + } + } + + int32_t numOfCols = pInfo->pDataBlock->info.numOfCols; + void** pCols = calloc(numOfCols, POINTER_BYTES); + SSchema* pSchema = calloc(numOfCols, sizeof(SSchema)); + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* p1 = taosArrayGet(pInfo->pDataBlock->pDataBlock, i); + pCols[i] = p1->pData; + pSchema[i].colId = p1->info.colId; + pSchema[i].bytes = p1->info.bytes; + pSchema[i].type = p1->info.type; + } + + __compar_fn_t comp = getKeyComparFunc(pSchema[pInfo->colIndex].type, pInfo->order); + taoscQSort(pCols, pSchema, numOfCols, pInfo->pDataBlock->info.rows, pInfo->colIndex, comp); + + tfree(pCols); + tfree(pSchema); + return (pInfo->pDataBlock->info.rows > 0)? pInfo->pDataBlock:NULL; +} + +SOperatorInfo *createOrderOperatorInfo(SExprInfo* pExpr, int32_t numOfOutput) { SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo)); { @@ -6011,106 +6080,6 @@ static SSDataBlock* doFill(void* param, bool* newgroup) { } } -static int32_t doMergeSDatablock(SSDataBlock* pDest, SSDataBlock* pSrc) { - assert(pSrc != NULL && pDest != NULL && pDest->info.numOfCols == pSrc->info.numOfCols); - - int32_t numOfCols = pSrc->info.numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock->pData, i); - SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock->pData, i); - - int32_t newSize = (pDest->info.rows + pSrc->info.rows) * pCol2->info.bytes; - char* tmp = realloc(pCol2->pData, newSize); - if (tmp != NULL) { - pCol2->pData = tmp; - int32_t offset = pCol2->info.bytes * pDest->info.rows; - memcpy(pCol2->pData + offset, pCol1->pData, pSrc->info.rows * pCol2->info.bytes); - } else { - return TSDB_CODE_VND_OUT_OF_MEMORY; - } - } - - return TSDB_CODE_SUCCESS; -} - -static SSDataBlock* doSort(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SOrderOperatorInfo* pInfo = pOperator->info; -// SSDataBlock* pRes = pInfo->pRes; - -// pRes->info.rows = 0; - SSDataBlock* pBlock = NULL; - while(1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - // start to flush data into disk and try do multiway merge sort - if (pBlock == NULL) { -// setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); -// pOperator->status = OP_EXEC_DONE; - break; - } - - int32_t code = doMergeSDatablock(pInfo->pDataBlock, pBlock); - if (code != TSDB_CODE_SUCCESS) { -// return code; - } - - /*SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex); - - int16_t bytes = pColInfoData->info.bytes; - int16_t type = pColInfoData->info.type; - - // ensure the output buffer size - SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, 0); - if (pRes->info.rows + pBlock->info.rows > pInfo->outputCapacity) { - int32_t newSize = pRes->info.rows + pBlock->info.rows; - char* tmp = realloc(pResultColInfoData->pData, newSize * bytes); - if (tmp == NULL) { - return NULL; - } else { - pResultColInfoData->pData = tmp; - pInfo->outputCapacity = newSize; - } - } - - for(int32_t i = 0; i < pBlock->info.rows; ++i) { - char* val = ((char*)pColInfoData->pData) + bytes * i; - if (isNull(val, type)) { - continue; - } - - size_t keyLen = 0; - if (IS_VAR_DATA_TYPE(pOperator->pExpr->base.colType)) { - tstr* var = (tstr*)(val); - keyLen = varDataLen(var); - } else { - keyLen = bytes; - } - - int dummy; - void* res = taosHashGet(pInfo->pSet, val, keyLen); - if (res == NULL) { - taosHashPut(pInfo->pSet, val, keyLen, &dummy, sizeof(dummy)); - char* start = pResultColInfoData->pData + bytes * pInfo->pRes->info.rows; - memcpy(start, val, bytes); - pRes->info.rows += 1; - } - } - - if (pRes->info.rows >= pInfo->threshold) { - break; - }*/ - } - - return (pInfo->pDataBlock->info.rows > 0)? pInfo->pDataBlock:NULL; -} - // todo set the attribute of query scan count static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr) { for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index cc47cc824b..c4f5d6efd5 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -12,7 +12,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "qExtbuffer.h" #include "os.h" #include "qAggMain.h" #include "queryLog.h" @@ -21,6 +20,8 @@ #include "taosmsg.h" #include "tulog.h" #include "qExecutor.h" +#include "qExtbuffer.h" +#include "tcompare.h" #define COLMODEL_GET_VAL(data, schema, allrow, rowId, colId) \ (data + (schema)->pFields[colId].offset * (allrow) + (rowId) * (schema)->pFields[colId].field.bytes) @@ -767,6 +768,60 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta free(buf); } +void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t index, __compar_fn_t compareFn) { + assert(numOfRows > 0 && numOfCols > 0 && index >= 0 && index < numOfCols); + + int32_t bytes = pSchema[index].bytes; + int32_t size = bytes + sizeof(int32_t); + + char* buf = calloc(1, size * numOfRows); + + for(int32_t i = 0; i < numOfRows; ++i) { + char* dest = buf + size * i; + memcpy(dest, pCols[index] + bytes * i, bytes); + *(int32_t*)(dest+bytes) = i; + } + + qsort(buf, numOfRows, size, compareFn); + + int32_t prevLength = 0; + char* p = NULL; + + for(int32_t i = 0; i < numOfCols; ++i) { + int32_t bytes1 = pSchema[i].bytes; + + if (i == index) { + for(int32_t j = 0; j < numOfRows; ++j){ + char* src = buf + (j * size); + char* dest = pCols[i] + (j * bytes1); + memcpy(dest, src, bytes1); + } + } else { + // make sure memory buffer is enough + if (prevLength < bytes1) { + char *tmp = realloc(p, bytes1 * numOfRows); + assert(tmp); + + p = tmp; + prevLength = bytes1; + } + + memcpy(p, pCols[i], bytes1 * numOfRows); + + for(int32_t j = 0; j < numOfRows; ++j){ + char* dest = pCols[i] + bytes1 * j; + + int32_t newPos = *(int32_t*)(buf + (j * size) + bytes); + char* src = p + (newPos * bytes1); + memcpy(dest, src, bytes1); + } + } + } + + tfree(buf); + tfree(p); +} + /* * deep copy of sschema */ diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c index e3326cc26b..e9022db503 100644 --- a/src/query/src/qPercentile.c +++ b/src/query/src/qPercentile.c @@ -237,7 +237,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, } pBucket->elemPerPage = (pBucket->bufPageSize - sizeof(tFilePage))/pBucket->bytes; - pBucket->comparFn = getKeyComparFunc(pBucket->type); + pBucket->comparFn = getKeyComparFunc(pBucket->type, TSDB_ORDER_ASC); pBucket->hashFunc = getHashFunc(pBucket->type); if (pBucket->hashFunc == NULL) { diff --git a/src/query/tests/cSortTest.cpp b/src/query/tests/cSortTest.cpp new file mode 100644 index 0000000000..aa5aa89afc --- /dev/null +++ b/src/query/tests/cSortTest.cpp @@ -0,0 +1,124 @@ +#include +#include + +#include "taos.h" +#include "tsdb.h" +#include "qExtbuffer.h" + +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +namespace { + int32_t comp(const void* p1, const void* p2) { + int32_t* x1 = (int32_t*) p1; + int32_t* x2 = (int32_t*) p2; + + if (*x1 == *x2) { + return 0; + } else { + return (*x1 > *x2)? 1:-1; + } + } + + int32_t comp1(const void* p1, const void* p2) { + int32_t ret = strncmp((char*) p1, (char*) p2, 20); + + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1:-1; + } + } +} + +TEST(testCase, colunmnwise_sort_test) { + // void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t index, __compar_fn_t compareFn) + void* pCols[2] = {0}; + + SSchema s[2] = {{0}}; + s[0].type = TSDB_DATA_TYPE_INT; + s[0].bytes = 4; + s[0].colId = 0; + strcpy(s[0].name, "col1"); + + s[1].type = TSDB_DATA_TYPE_BINARY; + s[1].bytes = 20; + s[1].colId = 1; + strcpy(s[1].name, "col2"); + + int32_t* p = (int32_t*) calloc(5, sizeof(int32_t)); + p[0] = 12; + p[1] = 8; + p[2] = 99; + p[3] = 7; + p[4] = 1; + + char* t1 = (char*) calloc(5, 20); + strcpy(t1, "abc"); + strcpy(t1 + 20, "def"); + strcpy(t1 + 40, "xyz"); + strcpy(t1 + 60, "klm"); + strcpy(t1 + 80, "hij"); + + pCols[0] = (char*) p; + pCols[1] = (char*) t1; + taoscQSort(reinterpret_cast(pCols), s, 2, 5, 0, comp); + + int32_t* px = (int32_t*) pCols[0]; + ASSERT_EQ(px[0], 1); + ASSERT_EQ(px[1], 7); + ASSERT_EQ(px[2], 8); + ASSERT_EQ(px[3], 12); + ASSERT_EQ(px[4], 99); + + char* px1 = (char*) pCols[1]; + ASSERT_STRCASEEQ(px1 + 20 * 0, "hij"); + ASSERT_STRCASEEQ(px1 + 20 * 1, "klm"); + ASSERT_STRCASEEQ(px1 + 20 * 2, "def"); + ASSERT_STRCASEEQ(px1 + 20 * 3, "abc"); + ASSERT_STRCASEEQ(px1 + 20 * 4, "xyz"); + + taoscQSort(pCols, s, 2, 5, 1, comp1); + px = (int32_t*) pCols[0]; + ASSERT_EQ(px[0], 12); + ASSERT_EQ(px[1], 8); + ASSERT_EQ(px[2], 1); + ASSERT_EQ(px[3], 7); + ASSERT_EQ(px[4], 99); + + px1 = (char*) pCols[1]; + ASSERT_STRCASEEQ(px1 + 20 * 0, "abc"); + ASSERT_STRCASEEQ(px1 + 20 * 1, "def"); + ASSERT_STRCASEEQ(px1 + 20 * 2, "hij"); + ASSERT_STRCASEEQ(px1 + 20 * 3, "klm"); + ASSERT_STRCASEEQ(px1 + 20 * 4, "xyz"); +} + +TEST(testCase, columnsort_test) { + SSchema field[1] = { + {TSDB_DATA_TYPE_INT, "k", sizeof(int32_t)}, + }; + + const int32_t num = 2000; + + int32_t *d = (int32_t *)malloc(sizeof(int32_t) * num); + for (int32_t i = 0; i < num; ++i) { + d[i] = i % 4; + } + + const int32_t numOfOrderCols = 1; + int32_t orderColIdx = 0; + SColumnModel *pModel = createColumnModel(field, 1, 1000); + tOrderDescriptor *pDesc = tOrderDesCreate(&orderColIdx, numOfOrderCols, pModel, 1); + + tColDataQSort(pDesc, num, 0, num - 1, (char *)d, 1); + + for (int32_t i = 0; i < num; ++i) { + printf("%d\t", d[i]); + } + printf("\n"); + + destroyColumnModel(pModel); +} \ No newline at end of file diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp index e5487a061d..2338adc2c5 100644 --- a/src/query/tests/unitTest.cpp +++ b/src/query/tests/unitTest.cpp @@ -1,6 +1,4 @@ -#include "os.h" #include -#include #include #include "taos.h" diff --git a/src/util/inc/tcompare.h b/src/util/inc/tcompare.h index 612ce7ede0..4861779acd 100644 --- a/src/util/inc/tcompare.h +++ b/src/util/inc/tcompare.h @@ -47,7 +47,7 @@ int WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, con int32_t doCompare(const char* a, const char* b, int32_t type, size_t size); -__compar_fn_t getKeyComparFunc(int32_t keyType); +__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order); __compar_fn_t getComparFunc(int32_t type, int32_t optr); diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 55ba14f84f..57689b7269 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -16,21 +16,17 @@ #include "os.h" #include "ttype.h" #include "tcompare.h" -#include "tarray.h" #include "hash.h" -int32_t compareInt32Val(const void *pLeft, const void *pRight) { - int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight); +int32_t compareInt8Val(const void *pLeft, const void *pRight) { + int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight); if (left > right) return 1; if (left < right) return -1; return 0; } -int32_t compareInt64Val(const void *pLeft, const void *pRight) { - int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight); - if (left > right) return 1; - if (left < right) return -1; - return 0; +int32_t compareInt8ValDesc(const void *pLeft, const void *pRight) { + return compareInt8Val(pRight, pLeft); } int32_t compareInt16Val(const void *pLeft, const void *pRight) { @@ -40,13 +36,32 @@ int32_t compareInt16Val(const void *pLeft, const void *pRight) { return 0; } -int32_t compareInt8Val(const void *pLeft, const void *pRight) { - int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight); +int32_t compareInt16ValDesc(const void* pLeft, const void* pRight) { + return compareInt16Val(pRight, pLeft); +} + +int32_t compareInt32Val(const void *pLeft, const void *pRight) { + int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight); if (left > right) return 1; if (left < right) return -1; return 0; } +int32_t compareInt32ValDesc(const void* pLeft, const void* pRight) { + return compareInt32Val(pRight, pLeft); +} + +int32_t compareInt64Val(const void *pLeft, const void *pRight) { + int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; +} + +int32_t compareInt64ValDesc(const void* pLeft, const void* pRight) { + return compareInt64Val(pRight, pLeft); +} + int32_t compareUint32Val(const void *pLeft, const void *pRight) { int32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight); if (left > right) return 1; @@ -54,6 +69,10 @@ int32_t compareUint32Val(const void *pLeft, const void *pRight) { return 0; } +int32_t compareUint32ValDesc(const void* pLeft, const void* pRight) { + return compareUint32Val(pRight, pLeft); +} + int32_t compareUint64Val(const void *pLeft, const void *pRight) { int64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight); if (left > right) return 1; @@ -61,6 +80,10 @@ int32_t compareUint64Val(const void *pLeft, const void *pRight) { return 0; } +int32_t compareUint64ValDesc(const void* pLeft, const void* pRight) { + return compareUint64Val(pRight, pLeft); +} + int32_t compareUint16Val(const void *pLeft, const void *pRight) { int16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight); if (left > right) return 1; @@ -68,6 +91,10 @@ int32_t compareUint16Val(const void *pLeft, const void *pRight) { return 0; } +int32_t compareUint16ValDesc(const void* pLeft, const void* pRight) { + return compareUint16Val(pRight, pLeft); +} + int32_t compareUint8Val(const void* pLeft, const void* pRight) { uint8_t left = GET_UINT8_VAL(pLeft), right = GET_UINT8_VAL(pRight); if (left > right) return 1; @@ -75,6 +102,10 @@ int32_t compareUint8Val(const void* pLeft, const void* pRight) { return 0; } +int32_t compareUint8ValDesc(const void* pLeft, const void* pRight) { + return compareUint8Val(pRight, pLeft); +} + int32_t compareFloatVal(const void *pLeft, const void *pRight) { float p1 = GET_FLOAT_VAL(pLeft); float p2 = GET_FLOAT_VAL(pRight); @@ -96,6 +127,10 @@ int32_t compareFloatVal(const void *pLeft, const void *pRight) { return FLT_GREATER(p1, p2) ? 1: -1; } +int32_t compareFloatValDesc(const void* pLeft, const void* pRight) { + return compareFloatVal(pRight, pLeft); +} + int32_t compareDoubleVal(const void *pLeft, const void *pRight) { double p1 = GET_DOUBLE_VAL(pLeft); double p2 = GET_DOUBLE_VAL(pRight); @@ -117,6 +152,10 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) { return FLT_GREATER(p1, p2) ? 1: -1; } +int32_t compareDoubleValDesc(const void* pLeft, const void* pRight) { + return compareDoubleVal(pRight, pLeft); +} + int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) { int32_t len1 = varDataLen(pLeft); int32_t len2 = varDataLen(pRight); @@ -133,6 +172,10 @@ int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) { } } +int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight) { + return compareLenPrefixedStr(pRight, pLeft); +} + int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) { int32_t len1 = varDataLen(pLeft); int32_t len2 = varDataLen(pRight); @@ -149,6 +192,10 @@ int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) { } } +int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) { + return compareLenPrefixedWStr(pRight, pLeft); +} + /* * Compare two strings * TSDB_MATCH: Match @@ -349,50 +396,50 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { return comparFn; } -__compar_fn_t getKeyComparFunc(int32_t keyType) { +__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { __compar_fn_t comparFn = NULL; switch (keyType) { case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_BOOL: - comparFn = compareInt8Val; + comparFn = (order == TSDB_ORDER_ASC)? compareInt8Val:compareInt8ValDesc; break; case TSDB_DATA_TYPE_SMALLINT: - comparFn = compareInt16Val; + comparFn = (order == TSDB_ORDER_ASC)? compareInt16Val:compareInt16ValDesc; break; case TSDB_DATA_TYPE_INT: - comparFn = compareInt32Val; + comparFn = (order == TSDB_ORDER_ASC)? compareInt32Val:compareInt32ValDesc; break; case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: - comparFn = compareInt64Val; + comparFn = (order == TSDB_ORDER_ASC)? compareInt64Val:compareInt64ValDesc; break; case TSDB_DATA_TYPE_FLOAT: - comparFn = compareFloatVal; + comparFn = (order == TSDB_ORDER_ASC)? compareFloatVal:compareFloatValDesc; break; case TSDB_DATA_TYPE_DOUBLE: - comparFn = compareDoubleVal; + comparFn = (order == TSDB_ORDER_ASC)? compareDoubleVal:compareDoubleValDesc; break; case TSDB_DATA_TYPE_UTINYINT: - comparFn = compareUint8Val; + comparFn = (order == TSDB_ORDER_ASC)? compareUint8Val:compareUint8ValDesc; break; case TSDB_DATA_TYPE_USMALLINT: - comparFn = compareUint16Val; + comparFn = (order == TSDB_ORDER_ASC)? compareUint16Val:compareUint16ValDesc; break; case TSDB_DATA_TYPE_UINT: - comparFn = compareUint32Val; + comparFn = (order == TSDB_ORDER_ASC)? compareUint32Val:compareUint32ValDesc; break; case TSDB_DATA_TYPE_UBIGINT: - comparFn = compareUint64Val; + comparFn = (order == TSDB_ORDER_ASC)? compareUint64Val:compareUint64ValDesc; break; case TSDB_DATA_TYPE_BINARY: - comparFn = compareLenPrefixedStr; + comparFn = (order == TSDB_ORDER_ASC)? compareLenPrefixedStr:compareLenPrefixedStrDesc; break; case TSDB_DATA_TYPE_NCHAR: - comparFn = compareLenPrefixedWStr; + comparFn = (order == TSDB_ORDER_ASC)? compareLenPrefixedWStr:compareLenPrefixedWStrDesc; break; default: - comparFn = compareInt32Val; + comparFn = (order == TSDB_ORDER_ASC)? compareInt32Val:compareInt32ValDesc; break; } diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index b464519ba6..98fd9c094c 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -54,7 +54,7 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _ pSkipList->keyFn = fn; pSkipList->seed = rand(); if (comparFn == NULL) { - pSkipList->comparFn = getKeyComparFunc(keyType); + pSkipList->comparFn = getKeyComparFunc(keyType, TSDB_ORDER_ASC); } else { pSkipList->comparFn = comparFn; } From 6400fe192cfcf3c656ee8900392a1a0e729ac722 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 5 Aug 2021 13:13:17 +0800 Subject: [PATCH 04/75] [td-225]fix compiler error. --- src/query/src/qExecutor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index f836d83730..5d8d2bb545 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5272,7 +5272,7 @@ static SSDataBlock* doSort(void* param, bool* newgroup) { pCols[i] = p1->pData; pSchema[i].colId = p1->info.colId; pSchema[i].bytes = p1->info.bytes; - pSchema[i].type = p1->info.type; + pSchema[i].type = (uint8_t) p1->info.type; } __compar_fn_t comp = getKeyComparFunc(pSchema[pInfo->colIndex].type, pInfo->order); From fbe535bd8be422cb2ddade9528302c5eab4335db Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 5 Aug 2021 16:27:13 +0800 Subject: [PATCH 05/75] [td-225] fix compiler error and invalid in query. --- src/client/src/tscSQLParser.c | 7 ++++--- src/common/src/tname.c | 3 +-- src/query/src/qExtbuffer.c | 6 +++--- tests/pytest/query/long_where_query.py | 6 +----- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 5739333886..df2f39ee87 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3736,7 +3736,8 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, if (pRight->tokenId != TK_SET || !serializeExprListToVariant(pRight->Expr.paramList, &pVal, colType, timePrecision)) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } - pColumnFilter->pz = (int64_t)calloc(1, pVal->nLen + 1); + + pColumnFilter->pz = (int64_t)calloc(1, pVal->nLen); pColumnFilter->len = pVal->nLen; pColumnFilter->filterstr = 1; memcpy((char *)(pColumnFilter->pz), (char *)(pVal->pz), pVal->nLen); @@ -8772,13 +8773,13 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf tfree(p); } -#if 0 +//#if 0 SQueryNode* p = qCreateQueryPlan(pQueryInfo); char* s = queryPlanToString(p); printf("%s\n", s); tfree(s); qDestroyQueryPlan(p); -#endif +//#endif return TSDB_CODE_SUCCESS; // Does not build query message here } diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 5da48b2e9a..aeade05df2 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -70,12 +70,11 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil memcpy(pFilter, src, sizeof(SColumnFilterInfo) * numOfFilters); for (int32_t j = 0; j < numOfFilters; ++j) { - if (pFilter[j].filterstr) { size_t len = (size_t) pFilter[j].len + 1 * TSDB_NCHAR_SIZE; pFilter[j].pz = (int64_t) calloc(1, len); - memcpy((char*)pFilter[j].pz, (char*)src[j].pz, (size_t)len); + memcpy((char*)pFilter[j].pz, (char*)src[j].pz, pFilter[j].len); } } diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index c4f5d6efd5..9f9347b327 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -778,7 +778,7 @@ void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOf for(int32_t i = 0; i < numOfRows; ++i) { char* dest = buf + size * i; - memcpy(dest, pCols[index] + bytes * i, bytes); + memcpy(dest, ((char*)pCols[index]) + bytes * i, bytes); *(int32_t*)(dest+bytes) = i; } @@ -793,7 +793,7 @@ void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOf if (i == index) { for(int32_t j = 0; j < numOfRows; ++j){ char* src = buf + (j * size); - char* dest = pCols[i] + (j * bytes1); + char* dest = (char*) pCols[i] + (j * bytes1); memcpy(dest, src, bytes1); } } else { @@ -809,7 +809,7 @@ void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOf memcpy(p, pCols[i], bytes1 * numOfRows); for(int32_t j = 0; j < numOfRows; ++j){ - char* dest = pCols[i] + bytes1 * j; + char* dest = (char*) pCols[i] + bytes1 * j; int32_t newPos = *(int32_t*)(buf + (j * size) + bytes); char* src = p + (newPos * bytes1); diff --git a/tests/pytest/query/long_where_query.py b/tests/pytest/query/long_where_query.py index 62e9533b62..9bb5f0b3d7 100644 --- a/tests/pytest/query/long_where_query.py +++ b/tests/pytest/query/long_where_query.py @@ -287,13 +287,9 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.error(sql) - endTime = time.time() print("total time %ds" % (endTime - startTime)) - - - - os.system("rm -rf query/long_where_query.py.sql") + #os.system("rm -rf query/long_where_query.py.sql") def stop(self): From d582adfa128064b05a3ac493df91f1a511174fb6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 5 Aug 2021 16:33:55 +0800 Subject: [PATCH 06/75] [td-225]disable print logic plan. --- src/client/src/tscSQLParser.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index df2f39ee87..399f2f17d1 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -8773,13 +8773,13 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf tfree(p); } -//#if 0 +#if 0 SQueryNode* p = qCreateQueryPlan(pQueryInfo); char* s = queryPlanToString(p); printf("%s\n", s); tfree(s); qDestroyQueryPlan(p); -//#endif +#endif return TSDB_CODE_SUCCESS; // Does not build query message here } From c0d80b875d146e16411b8ace491ce297e3e0e818 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 5 Aug 2021 18:09:59 +0800 Subject: [PATCH 07/75] fix crash issue --- src/client/src/tscProfile.c | 6 +++++- src/client/src/tscUtil.c | 3 +-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index 92ad9b7924..e307690c24 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -264,9 +264,12 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { snprintf(p, remainLen, "N/A"); } else { int32_t len; + if (pSql->pSubs != NULL && pSql->subState.states != NULL) { for (int32_t i = 0; i < pQdesc->numOfSub; ++i) { + SSqlObj* psub = pSql->pSubs[i]; + int64_t self = (psub != NULL)? psub->self:0; len = snprintf(p, remainLen, "[%d]0x%" PRIx64 "(%c) ", i, - pSql->pSubs[i]->self, + self, pSql->subState.states[i] ? 'C' : 'I'); if (len > remainLen) { break; @@ -274,6 +277,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { remainLen -= len; p += len; } + } } pQdesc->numOfSub = htonl(pQdesc->numOfSub); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 71aa75dafb..ecd65a7ae4 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1482,8 +1482,6 @@ void tscFreeSqlObj(SSqlObj* pSql) { tscFreeMetaSqlObj(&pSql->metaRid); tscFreeMetaSqlObj(&pSql->svgroupRid); - tscFreeSubobj(pSql); - SSqlCmd* pCmd = &pSql->cmd; int32_t cmd = pCmd->command; if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_GLOBALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT || @@ -1491,6 +1489,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { tscRemoveFromSqlList(pSql); } + tscFreeSubobj(pSql); pSql->signature = NULL; pSql->fp = NULL; tfree(pSql->sqlstr); From 24db3339cf1557f1d80d3e0f600e7b2f4a4d1b18 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 5 Aug 2021 18:18:15 +0800 Subject: [PATCH 08/75] [td-225]fix concurrent issue during building the heartbeat message. --- src/client/src/tscProfile.c | 45 +++++++++++++++++++------------------ src/client/src/tscUtil.c | 1 + 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index e307690c24..b75eb5716f 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -227,16 +227,16 @@ void tscKillStream(STscObj *pObj, uint32_t killId) { int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { SHeartBeatMsg *pHeartbeat = pMsg; + int allocedQueriesNum = pHeartbeat->numOfQueries; int allocedStreamsNum = pHeartbeat->numOfStreams; pHeartbeat->numOfQueries = 0; SQueryDesc *pQdesc = (SQueryDesc *)pHeartbeat->pData; - // We extract the lock to tscBuildHeartBeatMsg function. - int64_t now = taosGetTimestampMs(); SSqlObj *pSql = pObj->sqlList; + while (pSql) { /* * avoid sqlobj may not be correctly removed from sql list @@ -248,45 +248,46 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { } tstrncpy(pQdesc->sql, pSql->sqlstr, sizeof(pQdesc->sql)); - pQdesc->stime = htobe64(pSql->stime); - pQdesc->queryId = htonl(pSql->queryId); - //pQdesc->useconds = htobe64(pSql->res.useconds); + pQdesc->stime = htobe64(pSql->stime); + pQdesc->queryId = htonl(pSql->queryId); pQdesc->useconds = htobe64(now - pSql->stime); - pQdesc->qId = htobe64(pSql->res.qId); + pQdesc->qId = htobe64(pSql->res.qId); pQdesc->sqlObjId = htobe64(pSql->self); - pQdesc->pid = pHeartbeat->pid; - pQdesc->stableQuery = pSql->cmd.pQueryInfo->stableQuery; + pQdesc->pid = pHeartbeat->pid; pQdesc->numOfSub = pSql->subState.numOfSub; + pQdesc->stableQuery = pSql->cmd.pQueryInfo->stableQuery; char *p = pQdesc->subSqlInfo; int32_t remainLen = sizeof(pQdesc->subSqlInfo); if (pQdesc->numOfSub == 0) { snprintf(p, remainLen, "N/A"); } else { - int32_t len; if (pSql->pSubs != NULL && pSql->subState.states != NULL) { - for (int32_t i = 0; i < pQdesc->numOfSub; ++i) { - SSqlObj* psub = pSql->pSubs[i]; - int64_t self = (psub != NULL)? psub->self:0; - len = snprintf(p, remainLen, "[%d]0x%" PRIx64 "(%c) ", i, - self, - pSql->subState.states[i] ? 'C' : 'I'); - if (len > remainLen) { - break; + for (int32_t i = 0; i < pQdesc->numOfSub; ++i) { + SSqlObj *psub = pSql->pSubs[i]; + int64_t self = (psub != NULL)? psub->self : 0; + + int32_t len = snprintf(p, remainLen, "[%d]0x%" PRIx64 "(%c) ", i, self, pSql->subState.states[i] ? 'C' : 'I'); + if (len > remainLen) { + break; + } + + remainLen -= len; + p += len; } - remainLen -= len; - p += len; - } } } - pQdesc->numOfSub = htonl(pQdesc->numOfSub); + pQdesc->numOfSub = htonl(pQdesc->numOfSub); taosGetFqdn(pQdesc->fqdn); pHeartbeat->numOfQueries++; pQdesc++; + pSql = pSql->next; - if (pHeartbeat->numOfQueries >= allocedQueriesNum) break; + if (pHeartbeat->numOfQueries >= allocedQueriesNum) { + break; + } } pHeartbeat->numOfStreams = 0; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index ecd65a7ae4..a0e3c83a2e 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1490,6 +1490,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { } tscFreeSubobj(pSql); + pSql->signature = NULL; pSql->fp = NULL; tfree(pSql->sqlstr); From cf73e1f8afff74da0be97b30082068e6d5a41dfa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 5 Aug 2021 18:47:11 +0800 Subject: [PATCH 09/75] [td-225]add one retry case for subquery of super table. --- src/client/src/tscSubquery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 3816e6d619..eb7907f556 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -3032,7 +3032,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { assert(code == taos_errno(pSql)); - if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && (code != TSDB_CODE_TDB_INVALID_TABLE_ID)) { + if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && (code != TSDB_CODE_TDB_INVALID_TABLE_ID && code != TSDB_CODE_VND_INVALID_VGROUP_ID)) { tscError("0x%"PRIx64" sub:0x%"PRIx64" failed code:%s, retry:%d", pParentSql->self, pSql->self, tstrerror(code), trsupport->numOfRetry); int32_t sent = 0; From fbbc3f10f9e2aa6b90e31d5c10ce2571b9abdb1a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 5 Aug 2021 23:26:01 +0800 Subject: [PATCH 10/75] Update tname.c --- src/common/src/tname.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/src/tname.c b/src/common/src/tname.c index aeade05df2..40433d1f00 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -74,7 +74,7 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil size_t len = (size_t) pFilter[j].len + 1 * TSDB_NCHAR_SIZE; pFilter[j].pz = (int64_t) calloc(1, len); - memcpy((char*)pFilter[j].pz, (char*)src[j].pz, pFilter[j].len); + memcpy((char*)pFilter[j].pz, (char*)src[j].pz, (size_t) pFilter[j].len); } } From aa8b12e29bbf6b04a6f819e42103e91c460c8a5c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 Aug 2021 01:01:08 +0800 Subject: [PATCH 11/75] [TD-5797] support multi distinct --- src/client/src/tscSQLParser.c | 8 +- src/query/inc/qExecutor.h | 10 ++- src/query/src/qExecutor.c | 141 +++++++++++++++++++++------------- 3 files changed, 101 insertions(+), 58 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 5739333886..a6db366643 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1948,10 +1948,10 @@ bool isValidDistinctSql(SQueryInfo* pQueryInfo) { && (pQueryInfo->type & TSDB_QUERY_TYPE_TABLE_QUERY) != TSDB_QUERY_TYPE_TABLE_QUERY) { return false; } - if (tscNumOfExprs(pQueryInfo) == 1){ - return true; - } - return false; + //if (tscNumOfExprs(pQueryInfo) == 1){ + // return true; + //} + return true; } static bool hasNoneUserDefineExpr(SQueryInfo* pQueryInfo) { diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index ce70a9ba4a..8249a84e7f 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -508,13 +508,21 @@ typedef struct SStateWindowOperatorInfo { bool reptScan; } SStateWindowOperatorInfo ; +typedef struct SDistinctDataInfo { + int32_t index; + int32_t type; + int32_t bytes; +} SDistinctDataInfo; + typedef struct SDistinctOperatorInfo { SHashObj *pSet; SSDataBlock *pRes; bool recordNullVal; //has already record the null value, no need to try again int64_t threshold; int64_t outputCapacity; - int32_t colIndex; + int32_t totalBytes; + char* buf; + SArray* pDistinctDataInfo; } SDistinctOperatorInfo; struct SGlobalMerger; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 5323b4306f..fa54e9dc6b 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -44,6 +44,10 @@ #define SDATA_BLOCK_INITIALIZER (SDataBlockInfo) {{0}, 0} +#define MULTI_KEY_DELIM "-" + +#define HASH_CAPACITY_LIMIT 10000000 + #define TIME_WINDOW_COPY(_dst, _src) do {\ (_dst).skey = (_src).skey;\ (_dst).ekey = (_src).ekey;\ @@ -6109,6 +6113,8 @@ static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) { static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) { SDistinctOperatorInfo* pInfo = (SDistinctOperatorInfo*) param; taosHashCleanup(pInfo->pSet); + tfree(pInfo->buf); + taosArrayDestroy(pInfo->pDistinctDataInfo); pInfo->pRes = destroyOutputBuf(pInfo->pRes); } @@ -6600,20 +6606,65 @@ SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInf return pOperator; } +static bool initMultiDistinctInfo(SDistinctOperatorInfo *pInfo, SOperatorInfo* pOperator, SSDataBlock *pBlock) { + if (taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput) { + // distinct info already inited + return true; + } + for (int i = 0; i < pOperator->numOfOutput; i++) { + pInfo->totalBytes += pOperator->pExpr[i].base.colBytes; + } + for (int i = 0; i < pOperator->numOfOutput; i++) { + int numOfBlock = taosArrayGetSize(pBlock->pDataBlock); + assert(i < numOfBlock); + for (int j = 0; j < numOfBlock; j++) { + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, j); + if (pColDataInfo->info.colId == pOperator->pExpr[i].base.resColId) { + SDistinctDataInfo item = {.index = j, .type = pColDataInfo->info.type, .bytes = pColDataInfo->info.bytes}; + taosArrayInsert(pInfo->pDistinctDataInfo, i, &item); + } + } + } + pInfo->totalBytes += strlen(MULTI_KEY_DELIM) * (pOperator->numOfOutput); + pInfo->buf = calloc(1, pInfo->totalBytes); + return taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput ? true : false; +} +static void buildMultiDistinctKey(SDistinctOperatorInfo *pInfo, SSDataBlock *pBlock, int32_t rowId) { + char *p = pInfo->buf; + memset(p, 0, pInfo->totalBytes); + + for (int i = 0; i < taosArrayGetSize(pInfo->pDistinctDataInfo); i++) { + SDistinctDataInfo* pDistDataInfo = (SDistinctDataInfo *)taosArrayGet(pInfo->pDistinctDataInfo, i); + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index); + char *val = ((char *)pColDataInfo->pData) + pColDataInfo->info.bytes * rowId; + if (isNull(val, pDistDataInfo->type)) { + p += pDistDataInfo->bytes; + continue; + } + if (IS_VAR_DATA_TYPE(pDistDataInfo->type)) { + memcpy(p, varDataVal(val), varDataLen(val)); + p += varDataLen(val); + } else { + memcpy(p, val, pDistDataInfo->bytes); + p += pDistDataInfo->bytes; + } + memcpy(p, MULTI_KEY_DELIM, strlen(MULTI_KEY_DELIM)); + p += strlen(MULTI_KEY_DELIM); + } +} static SSDataBlock* hashDistinct(void* param, bool* newgroup) { SOperatorInfo* pOperator = (SOperatorInfo*) param; if (pOperator->status == OP_EXEC_DONE) { return NULL; } - - + SDistinctOperatorInfo* pInfo = pOperator->info; SSDataBlock* pRes = pInfo->pRes; - pRes->info.rows = 0; SSDataBlock* pBlock = NULL; + while(1) { publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); @@ -6624,63 +6675,44 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) { pOperator->status = OP_EXEC_DONE; break; } - if (pInfo->colIndex == -1) { - for (int i = 0; i < taosArrayGetSize(pBlock->pDataBlock); i++) { - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, i); - if (pColDataInfo->info.colId == pOperator->pExpr[0].base.resColId) { - pInfo->colIndex = i; - break; - } - } - } - if (pInfo->colIndex == -1) { + if (!initMultiDistinctInfo(pInfo, pOperator, pBlock)) { setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); pOperator->status = OP_EXEC_DONE; - return NULL; + break; } - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex); - - int16_t bytes = pColInfoData->info.bytes; - int16_t type = pColInfoData->info.type; - - // ensure the output buffer size - SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, 0); + // ensure result output buf if (pRes->info.rows + pBlock->info.rows > pInfo->outputCapacity) { int32_t newSize = pRes->info.rows + pBlock->info.rows; - char* tmp = realloc(pResultColInfoData->pData, newSize * bytes); - if (tmp == NULL) { - return NULL; - } else { - pResultColInfoData->pData = tmp; - pInfo->outputCapacity = newSize; + for (int i = 0; i < taosArrayGetSize(pRes->pDataBlock); i++) { + SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, i); + SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, i); + char* tmp = realloc(pResultColInfoData->pData, newSize * pDistDataInfo->bytes); + if (tmp == NULL) { + return NULL; + } else { + pResultColInfoData->pData = tmp; + } } + pInfo->outputCapacity = newSize; } - for(int32_t i = 0; i < pBlock->info.rows; ++i) { - char* val = ((char*)pColInfoData->pData) + bytes * i; - if (isNull(val, type)) { - continue; - } - char* p = val; - size_t keyLen = 0; - if (IS_VAR_DATA_TYPE(pOperator->pExpr->base.colType)) { - tstr* var = (tstr*)(val); - p = var->data; - keyLen = varDataLen(var); - } else { - keyLen = bytes; - } + for (int32_t i = 0; i < pBlock->info.rows; i++) { + buildMultiDistinctKey(pInfo, pBlock, i); + if (taosHashGet(pInfo->pSet, pInfo->buf, pInfo->totalBytes) == NULL) { + int32_t dummy; + taosHashPut(pInfo->pSet, pInfo->buf, pInfo->totalBytes, &dummy, sizeof(dummy)); + for (int j = 0; j < taosArrayGetSize(pRes->pDataBlock); j++) { + SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, j); // distinct meta info + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index); //src + SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, j); // dist - int dummy; - void* res = taosHashGet(pInfo->pSet, p, keyLen); - if (res == NULL) { - taosHashPut(pInfo->pSet, p, keyLen, &dummy, sizeof(dummy)); - char* start = pResultColInfoData->pData + bytes * pInfo->pRes->info.rows; - memcpy(start, val, bytes); + char* val = ((char*)pColInfoData->pData) + pDistDataInfo->bytes * i; + char *start = pResultColInfoData->pData + pDistDataInfo->bytes * pInfo->pRes->info.rows; + memcpy(start, val, pDistDataInfo->bytes); + } pRes->info.rows += 1; - } + } } - if (pRes->info.rows >= pInfo->threshold) { break; } @@ -6691,11 +6723,14 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) { SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { SDistinctOperatorInfo* pInfo = calloc(1, sizeof(SDistinctOperatorInfo)); - pInfo->colIndex = -1; - pInfo->threshold = 10000000; // distinct result threshold - pInfo->outputCapacity = 4096; - pInfo->pSet = taosHashInit(64, taosGetDefaultHashFunction(pExpr->base.colType), false, HASH_NO_LOCK); + pInfo->totalBytes = 0; + pInfo->buf = NULL; + pInfo->threshold = HASH_CAPACITY_LIMIT; // distinct result threshold + pInfo->outputCapacity = 4096; + pInfo->pDistinctDataInfo = taosArrayInit(numOfOutput, sizeof(SDistinctDataInfo)); + pInfo->pSet = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); pInfo->pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) pInfo->outputCapacity); + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "DistinctOperator"; From 39a96b3ebd6fb2179afc376461cb94bf39e0d235 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 6 Aug 2021 10:11:44 +0800 Subject: [PATCH 12/75] [td-225] fix compiler error. --- src/query/src/qExecutor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index e75b2ae12d..e9a1089268 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5290,7 +5290,7 @@ SOperatorInfo *createOrderOperatorInfo(SExprInfo* pExpr, int32_t numOfOutput) { SSDataBlock* pDataBlock = calloc(1, sizeof(SSDataBlock)); pDataBlock->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); for(int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData col = {0}; + SColumnInfoData col = {{0}}; col.info.bytes = pExpr->base.resBytes; col.info.colId = pExpr->base.resColId; col.info.type = pExpr->base.resType; From aa60eb572205b002ff4367c823a5bd1152f32cbb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 6 Aug 2021 14:34:23 +0800 Subject: [PATCH 13/75] [td-225]fix bug found by the regression test. --- src/client/src/tscAsync.c | 3 +-- src/client/src/tscServer.c | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index f39169c193..1c198fb8c6 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -346,7 +346,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (pSql->pStream == NULL) { SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT)) { + if (pQueryInfo != NULL && TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT)) { tscDebug("0x%" PRIx64 " continue parse sql after get table-meta", pSql->self); code = tsParseSql(pSql, false); @@ -376,7 +376,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { } else { if (pSql->retryReason != TSDB_CODE_SUCCESS) { tscDebug("0x%" PRIx64 " update cached table-meta, re-validate sql statement and send query again", pSql->self); - tscResetSqlCmd(pCmd, false); pSql->retryReason = TSDB_CODE_SUCCESS; } else { tscDebug("0x%" PRIx64 " cached table-meta, continue validate sql statement and send query", pSql->self); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index b2ed942b5e..b278235269 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2241,6 +2241,10 @@ int tscProcessMultiTableMetaRsp(SSqlObj *pSql) { pMsg = buf; } + if (pParentCmd->pTableMetaMap == NULL) { + pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + } + for (int32_t i = 0; i < pMultiMeta->numOfTables; i++) { STableMetaMsg *pMetaMsg = (STableMetaMsg *)pMsg; int32_t code = tableMetaMsgConvert(pMetaMsg); From b251e7b35953d913471e21fc66b39dfbfacd2c69 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 6 Aug 2021 23:05:30 +0800 Subject: [PATCH 14/75] [td-225]fix bug found by regression test. --- src/client/src/tscProfile.c | 10 ++++++++-- src/inc/taosmsg.h | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index b75eb5716f..f63e908dc0 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -18,11 +18,11 @@ #include "tsclient.h" #include "tsocket.h" #include "ttimer.h" -#include "tutil.h" #include "taosmsg.h" #include "tcq.h" #include "taos.h" +#include "tscUtil.h" void tscSaveSlowQueryFp(void *handle, void *tmrId); TAOS *tscSlowQueryConn = NULL; @@ -255,13 +255,19 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { pQdesc->sqlObjId = htobe64(pSql->self); pQdesc->pid = pHeartbeat->pid; pQdesc->numOfSub = pSql->subState.numOfSub; - pQdesc->stableQuery = pSql->cmd.pQueryInfo->stableQuery; char *p = pQdesc->subSqlInfo; int32_t remainLen = sizeof(pQdesc->subSqlInfo); if (pQdesc->numOfSub == 0) { snprintf(p, remainLen, "N/A"); } else { + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); + if (pQueryInfo != NULL) { + pQdesc->stableQuery = (pQueryInfo->stableQuery)?1:0; + } else { + pQdesc->stableQuery = 0; + } + if (pSql->pSubs != NULL && pSql->subState.states != NULL) { for (int32_t i = 0; i < pQdesc->numOfSub; ++i) { SSqlObj *psub = pSql->pSubs[i]; diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index c2918cfdf7..2af4e4857a 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -877,7 +877,7 @@ typedef struct { uint64_t sqlObjId; int32_t pid; char fqdn[TSDB_FQDN_LEN]; - bool stableQuery; + uint8_t stableQuery; int32_t numOfSub; char subSqlInfo[TSDB_SHOW_SUBQUERY_LEN]; //include subqueries' index, Obj IDs and states(C-complete/I-imcomplete) } SQueryDesc; From b46edae0b1237beda20138d5d8a873881bf6aea9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 7 Aug 2021 14:03:30 +0800 Subject: [PATCH 15/75] [td-225]fix a race condition. --- src/client/src/tscProfile.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index f63e908dc0..b00138b4c4 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -256,17 +256,20 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { pQdesc->pid = pHeartbeat->pid; pQdesc->numOfSub = pSql->subState.numOfSub; + // todo race condition + pQdesc->stableQuery = 0; + char *p = pQdesc->subSqlInfo; int32_t remainLen = sizeof(pQdesc->subSqlInfo); if (pQdesc->numOfSub == 0) { snprintf(p, remainLen, "N/A"); } else { - SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); - if (pQueryInfo != NULL) { - pQdesc->stableQuery = (pQueryInfo->stableQuery)?1:0; - } else { - pQdesc->stableQuery = 0; - } +// SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); +// if (pQueryInfo != NULL) { +// pQdesc->stableQuery = (pQueryInfo->stableQuery)?1:0; +// } else { +// pQdesc->stableQuery = 0; +// } if (pSql->pSubs != NULL && pSql->subState.states != NULL) { for (int32_t i = 0; i < pQdesc->numOfSub; ++i) { From 0fc9f7e3f30b7c6b2528b1d15fd7116212f04d30 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 7 Aug 2021 16:33:34 +0800 Subject: [PATCH 16/75] [TD-5797] support distict multi column --- src/common/src/tglobal.c | 14 ++++++++++++++ src/query/src/qExecutor.c | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index a58303e9fc..b8d12f7b01 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -84,6 +84,9 @@ int32_t tsMaxNumOfOrderedResults = 100000; // 10 ms for sliding time, the value will changed in case of time precision changed int32_t tsMinSlidingTime = 10; +// the maxinum number of distict query result +int32_t tsMaxNumOfDistinctResults = 1000 * 10000; + // 1 us for interval time range, changed accordingly int32_t tsMinIntervalTime = 1; @@ -541,6 +544,17 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "maxNumOfDistinctRes"; + cfg.ptr = &tsMaxNumOfDistinctResults; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; + cfg.minValue = 10*10000; + cfg.maxValue = 10000*10000; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "numOfMnodes"; cfg.ptr = &tsNumOfMnodes; cfg.valType = TAOS_CFG_VTYPE_INT32; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index fa54e9dc6b..219e9ad1e7 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3394,6 +3394,7 @@ void setDefaultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, i SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; int64_t tid = 0; + pRuntimeEnv->keyBuf = realloc(pRuntimeEnv->keyBuf, sizeof(tid) + sizeof(int64_t) + POINTER_BYTES); SResultRow* pRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, uid); for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { @@ -6725,7 +6726,7 @@ SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperat SDistinctOperatorInfo* pInfo = calloc(1, sizeof(SDistinctOperatorInfo)); pInfo->totalBytes = 0; pInfo->buf = NULL; - pInfo->threshold = HASH_CAPACITY_LIMIT; // distinct result threshold + pInfo->threshold = tsMaxNumOfDistinctResults; // distinct result threshold pInfo->outputCapacity = 4096; pInfo->pDistinctDataInfo = taosArrayInit(numOfOutput, sizeof(SDistinctDataInfo)); pInfo->pSet = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); From 9c9aae1632cb72487bb55868a27f59079a9aefa6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 7 Aug 2021 17:20:41 +0800 Subject: [PATCH 17/75] [TD-5797] support distict multi column --- src/common/inc/tglobal.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 7290db6ec9..fcdfeafefe 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -59,6 +59,7 @@ extern char tsLocale[]; extern char tsCharset[]; // default encode string extern int8_t tsEnableCoreFile; extern int32_t tsCompressMsgSize; +extern int32_t tsMaxNumOfDistinctResults; extern char tsTempDir[]; //query buffer management From c68ae361b2eb7875f3cab6de112e9295bf4ae99b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 8 Aug 2021 04:16:18 +0800 Subject: [PATCH 18/75] [TD-5799] self test distinct --- tests/script/general/parser/distinct.sim | 88 ++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 tests/script/general/parser/distinct.sim diff --git a/tests/script/general/parser/distinct.sim b/tests/script/general/parser/distinct.sim new file mode 100644 index 0000000000..e0eb74b5a5 --- /dev/null +++ b/tests/script/general/parser/distinct.sim @@ -0,0 +1,88 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5 +system sh/exec.sh -n dnode1 -s start +sleep 100 +sql connect + +$dbPrefix = sav_db +$tbPrefix = sav_tb +$stbPrefix = sav_stb +$tbNum = 20 +$rowNum = 10 +$totalNum = $tbNum * $rowNum +$ts0 = 1537146000000 +$delta = 600000 +print ========== alter.sim +$i = 0 +$db = $dbPrefix +$stb = $stbPrefix + +sql drop database if exists $db +sql create database $db +sql use $db +print ====== create tables +sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 int) + +$i = 0 +$ts = $ts0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $stb tags( $i , 0 ) + $i = $i + 1 + sql insert into $tb values('2015-08-18 00:00:00', 1); + sql insert into $tb values('2015-08-18 00:06:00', 2); + sql insert into $tb values('2015-08-18 00:12:00', 3); + sql insert into $tb values('2015-08-18 00:18:00', 4); + sql insert into $tb values('2015-08-18 00:24:00', 5); + sql insert into $tb values('2015-08-18 00:30:00', 6); +endw +$i = 0 +$tb = $tbPrefix . $i + +print ====== table created + +#### select distinct tag +sql select distinct t1 from $stb +if $rows != $tbNum then + return -1 +endi + +#### select distinct tag +sql select distinct t2 from $stb +if $rows != 1 then + print $rows + return -1 +endi + +#### select multi normal column +sql select distinct ts, c1 from $stb +if $rows != 6 then + return -1 +endi + +#### select multi column +sql select distinct ts from $stb +if $rows != 6 then + return -1 +endi + +### select multi normal column +### select distinct multi column on sub table + +sql select distinct ts, c1 from $tb +if $rows != 6 then + return -1 +endi + + +### select distinct +sql drop database $db +sql show databases +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 8e48713565b78eecd8d6e7a323175b135f7f5c05 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 8 Aug 2021 07:15:15 +0800 Subject: [PATCH 19/75] [TD-5799] fix compile error --- src/query/src/qExecutor.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 219e9ad1e7..f0e75872f1 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6616,7 +6616,7 @@ static bool initMultiDistinctInfo(SDistinctOperatorInfo *pInfo, SOperatorInfo* p pInfo->totalBytes += pOperator->pExpr[i].base.colBytes; } for (int i = 0; i < pOperator->numOfOutput; i++) { - int numOfBlock = taosArrayGetSize(pBlock->pDataBlock); + int numOfBlock = (int)(taosArrayGetSize(pBlock->pDataBlock)); assert(i < numOfBlock); for (int j = 0; j < numOfBlock; j++) { SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, j); @@ -6626,7 +6626,7 @@ static bool initMultiDistinctInfo(SDistinctOperatorInfo *pInfo, SOperatorInfo* p } } } - pInfo->totalBytes += strlen(MULTI_KEY_DELIM) * (pOperator->numOfOutput); + pInfo->totalBytes += (int32_t)strlen(MULTI_KEY_DELIM) * (pOperator->numOfOutput); pInfo->buf = calloc(1, pInfo->totalBytes); return taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput ? true : false; } From 50d6fef714559e17b7bfc10a912a215fee27ff71 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 9 Aug 2021 14:17:47 +0800 Subject: [PATCH 20/75] [td-225] fix the bug that failed to record the total elapsed time in the log file. --- src/query/src/qExecutor.c | 5 ++++- src/query/src/queryMain.c | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index e9a1089268..3d5e19ccd3 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2874,7 +2874,7 @@ void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFi int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { *status = BLK_DATA_NO_NEEDED; - pBlock->pDataBlock = NULL; + pBlock->pDataBlock = NULL; pBlock->pBlockStatis = NULL; SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -2884,6 +2884,9 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa SQInfo* pQInfo = pRuntimeEnv->qinfo; SQueryCostInfo* pCost = &pQInfo->summary; + pCost->totalBlocks += 1; + pCost->totalRows += pBlock->info.rows; + if (pRuntimeEnv->pTsBuf != NULL) { (*status) = BLK_DATA_ALL_NEEDED; diff --git a/src/query/src/queryMain.c b/src/query/src/queryMain.c index 7d30f7c668..878698752f 100644 --- a/src/query/src/queryMain.c +++ b/src/query/src/queryMain.c @@ -199,7 +199,6 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi return code; } - bool qTableQuery(qinfo_t qinfo, uint64_t *qId) { SQInfo *pQInfo = (SQInfo *)qinfo; assert(pQInfo && pQInfo->signature == pQInfo); @@ -240,7 +239,11 @@ bool qTableQuery(qinfo_t qinfo, uint64_t *qId) { bool newgroup = false; publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_BEFORE_OPERATOR_EXEC); + + int64_t st = taosGetTimestampUs(); pRuntimeEnv->outputBuf = pRuntimeEnv->proot->exec(pRuntimeEnv->proot, &newgroup); + pQInfo->summary.elapsedTime += (taosGetTimestampUs() - st); + publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_AFTER_OPERATOR_EXEC); pRuntimeEnv->resultInfo.total += GET_NUM_OF_RESULTS(pRuntimeEnv); From 2855bd73902edddf908d832d7e531e772c32d89e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 9 Aug 2021 14:29:15 +0800 Subject: [PATCH 21/75] [td-225]update the performance log info. --- src/query/src/qExecutor.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3d5e19ccd3..9c8219c559 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3310,12 +3310,11 @@ void copyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, int32_t threshold, SSDataBl } } - // enough results in data buffer, return - if (pBlock->info.rows >= threshold) { - break; - } + // enough results in data buffer, return + if (pBlock->info.rows >= threshold) { + break; } - + } } static void updateTableQueryInfoForReverseScan(STableQueryInfo *pTableQueryInfo) { @@ -5694,11 +5693,15 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { + int64_t st = taosGetTimestampUs(); copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } + SQInfo* pQInfo = pRuntimeEnv->qinfo; + pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st); + return pIntervalInfo->pRes; } @@ -5731,11 +5734,15 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { doCloseAllTimeWindow(pRuntimeEnv); setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + int64_t st = taosGetTimestampUs(); copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } + SQInfo* pQInfo = pRuntimeEnv->qinfo; + pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st); + return pIntervalInfo->pRes; } From 94b26bfea9b195bf6a8d973b7f393298ce1467cd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 9 Aug 2021 19:25:51 +0800 Subject: [PATCH 22/75] [td-225] --- src/client/inc/tscUtil.h | 6 ++-- src/client/inc/tsclient.h | 2 +- src/client/src/tscParseInsert.c | 4 +-- src/client/src/tscPrepare.c | 2 +- src/client/src/tscSQLParser.c | 2 +- src/client/src/tscServer.c | 10 ++---- src/client/src/tscStream.c | 62 ++++++++++++++++----------------- src/client/src/tscSubquery.c | 15 +++----- src/client/src/tscUtil.c | 51 ++++++++------------------- 9 files changed, 61 insertions(+), 93 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index b3674a7bf5..401da65908 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -220,7 +220,7 @@ void tscExprDestroy(SArray* pExprInfo); int32_t createProjectionExpr(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SExprInfo*** pExpr, int32_t* num); -void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta); +void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta, uint64_t id); SColumn* tscColumnClone(const SColumn* src); void tscColumnCopy(SColumn* pDest, const SColumn* pSrc); @@ -318,7 +318,7 @@ void tscPrintSelNodeList(SSqlObj* pSql, int32_t subClauseIndex); bool hasMoreVnodesToTry(SSqlObj *pSql); bool hasMoreClauseToTry(SSqlObj* pSql); -void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta); +void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeCachedMeta, uint64_t id); void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp); @@ -356,7 +356,7 @@ char* strdup_throw(const char* str); bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src); SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg); -void tscRemoveTableMetaBuf(STableMetaInfo* pTableMetaInfo, uint64_t id); +void tscRemoveCachedTableMeta(STableMetaInfo* pTableMetaInfo, uint64_t id); #ifdef __cplusplus } diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index ac5adcbbb4..91619f2d39 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -368,7 +368,7 @@ void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBloc void handleDownstreamOperator(SSqlObj** pSqlList, int32_t numOfUpstream, SQueryInfo* px, SSqlObj* pParent); void destroyTableNameList(SInsertStatementParam* pInsertParam); -void tscResetSqlCmd(SSqlCmd *pCmd, bool removeMeta); +void tscResetSqlCmd(SSqlCmd *pCmd, bool removeMeta, uint64_t id); /** * free query result of the sql object diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 73e4f898c8..6b82a1ef17 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1922,7 +1922,7 @@ int tsParseSql(SSqlObj *pSql, bool initial) { if (pSql->parseRetry < 1 && (ret == TSDB_CODE_TSC_SQL_SYNTAX_ERROR || ret == TSDB_CODE_TSC_INVALID_OPERATION)) { tscDebug("0x%"PRIx64 " parse insert sql statement failed, code:%s, clear meta cache and retry ", pSql->self, tstrerror(ret)); - tscResetSqlCmd(pCmd, true); + tscResetSqlCmd(pCmd, true, pSql->self); pSql->parseRetry++; if ((ret = tsInsertInitialCheck(pSql)) == TSDB_CODE_SUCCESS) { @@ -1939,7 +1939,7 @@ int tsParseSql(SSqlObj *pSql, bool initial) { if (ret == TSDB_CODE_TSC_INVALID_OPERATION && pSql->parseRetry < 1 && sqlInfo.type == TSDB_SQL_SELECT) { tscDebug("0x%"PRIx64 " parse query sql statement failed, code:%s, clear meta cache and retry ", pSql->self, tstrerror(ret)); - tscResetSqlCmd(pCmd, true); + tscResetSqlCmd(pCmd, true, pSql->self); pSql->parseRetry++; ret = tscValidateSqlInfo(pSql, &sqlInfo); diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 2c2a299549..9743c2837d 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1694,7 +1694,7 @@ int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags if (taosHashGetSize(pCmd->insertParam.pTableBlockHashList) > 0) { SHashObj* hashList = pCmd->insertParam.pTableBlockHashList; pCmd->insertParam.pTableBlockHashList = NULL; - tscResetSqlCmd(pCmd, false); + tscResetSqlCmd(pCmd, false, pSql->self); pCmd->insertParam.pTableBlockHashList = hashList; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 399f2f17d1..8f8ed6ae5e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -8483,7 +8483,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf } if (pSqlNode->from->type == SQL_NODE_FROM_SUBQUERY) { - clearAllTableMetaInfo(pQueryInfo, false); + clearAllTableMetaInfo(pQueryInfo, false, pSql->self); pQueryInfo->numOfTables = 0; // parse the subquery in the first place diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index b278235269..69c3357230 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2581,7 +2581,7 @@ int tscProcessDropDbRsp(SSqlObj *pSql) { int tscProcessDropTableRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - tscRemoveTableMetaBuf(pTableMetaInfo, pSql->self); + tscRemoveCachedTableMeta(pTableMetaInfo, pSql->self); tfree(pTableMetaInfo->pTableMeta); return 0; } @@ -2967,13 +2967,9 @@ int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { // remove stored tableMeta info in hash table - tscRemoveTableMetaBuf(pTableMetaInfo, pSql->self); - tscResetSqlCmd(pCmd, true); + tscResetSqlCmd(pCmd, true, pSql->self); -// pCmd->pTableMetaMap = tscCleanupTableMetaMap(pCmd->pTableMetaMap); -// pCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - - SArray* pNameList = taosArrayInit(1, POINTER_BYTES); + SArray* pNameList = taosArrayInit(1, POINTER_BYTES); SArray* vgroupList = taosArrayInit(1, POINTER_BYTES); char* n = strdup(name); diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 2c4bc5f764..9f2b79e891 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -113,7 +113,7 @@ static void doLaunchQuery(void* param, TAOS_RES* tres, int32_t code) { pQueryInfo->command = TSDB_SQL_SELECT; - pSql->fp = tscProcessStreamQueryCallback; + pSql->fp = tscProcessStreamQueryCallback; pSql->fetchFp = tscProcessStreamQueryCallback; executeQuery(pSql, pQueryInfo); tscIncStreamExecutionCount(pStream); @@ -142,6 +142,7 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) { if(pSql == NULL) { return ; } + SQueryInfo* pQueryInfo = tscGetQueryInfo(&pSql->cmd); tscDebug("0x%"PRIx64" add into timer", pSql->self); @@ -186,14 +187,16 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) { } // launch stream computing in a new thread - SSchedMsg schedMsg = { 0 }; - schedMsg.fp = tscProcessStreamLaunchQuery; + SSchedMsg schedMsg = {0}; + schedMsg.fp = tscProcessStreamLaunchQuery; schedMsg.ahandle = pStream; schedMsg.thandle = (void *)1; - schedMsg.msg = NULL; + schedMsg.msg = NULL; taosScheduleTask(tscQhandle, &schedMsg); } +static void cbParseSql(void* param, TAOS_RES* res, int code); + static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOfRows) { SSqlStream *pStream = (SSqlStream *)param; if (tres == NULL || numOfRows < 0) { @@ -201,24 +204,26 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf tscError("0x%"PRIx64" stream:%p, query data failed, code:0x%08x, retry in %" PRId64 "ms", pStream->pSql->self, pStream, numOfRows, retryDelay); - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0); + SSqlObj* pSql = pStream->pSql; - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); + tscFreeSqlResult(pSql); + tscFreeSubobj(pSql); + tfree(pSql->pSubs); + pSql->subState.numOfSub = 0; - taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + int32_t code = tsParseSql(pSql, true); + if (code == TSDB_CODE_SUCCESS) { + cbParseSql(pStream, pSql, code); + } else if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + tscDebug("0x%"PRIx64" CQ taso_open_stream IN Process", pSql->self); + } else { + tscError("0x%"PRIx64" open stream failed, code:%s", pSql->self, tstrerror(code)); + taosReleaseRef(tscObjRef, pSql->self); + free(pStream); + } - tfree(pTableMetaInfo->pTableMeta); - - tscFreeSqlResult(pStream->pSql); - tscFreeSubobj(pStream->pSql); - tfree(pStream->pSql->pSubs); - pStream->pSql->subState.numOfSub = 0; - - pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); - - tscSetRetryTimer(pStream, pStream->pSql, retryDelay); - return; +// tscSetRetryTimer(pStream, pStream->pSql, retryDelay); +// return; } taos_fetch_rows_a(tres, tscProcessStreamRetrieveResult, param); @@ -555,7 +560,6 @@ static void tscCreateStream(void *param, TAOS_RES *res, int code) { if (code != TSDB_CODE_SUCCESS) { pSql->res.code = code; tscError("0x%"PRIx64" open stream failed, sql:%s, reason:%s, code:%s", pSql->self, pSql->sqlstr, pCmd->payload, tstrerror(code)); - pStream->fp(pStream->param, NULL, NULL); return; } @@ -582,9 +586,10 @@ static void tscCreateStream(void *param, TAOS_RES *res, int code) { // set stime with ltime if ltime > stime const char* dstTable = pStream->dstTable? pStream->dstTable: ""; - tscDebug(" CQ table=%s ltime is %"PRId64, dstTable, pStream->ltime); + tscDebug("0x%"PRIx64" CQ table %s ltime is %"PRId64, pSql->self, dstTable, pStream->ltime); + if(pStream->ltime != INT64_MIN && pStream->ltime > pStream->stime) { - tscWarn(" CQ set stream %s stime=%"PRId64" replace with ltime=%"PRId64" if ltime>0 ", dstTable, pStream->stime, pStream->ltime); + tscWarn("0x%"PRIx64" CQ set stream %s stime=%"PRId64" replace with ltime=%"PRId64" if ltime > 0", pSql->self, dstTable, pStream->stime, pStream->ltime); pStream->stime = pStream->ltime; } @@ -592,7 +597,6 @@ static void tscCreateStream(void *param, TAOS_RES *res, int code) { pCmd->command = TSDB_SQL_SELECT; tscAddIntoStreamList(pStream); - taosTmrReset(tscProcessStreamTimer, (int32_t)starttime, pStream, tscTmr, &pStream->pTimer); tscDebug("0x%"PRIx64" stream:%p is opened, query on:%s, interval:%" PRId64 ", sliding:%" PRId64 ", first launched in:%" PRId64 ", sql:%s", pSql->self, @@ -659,10 +663,9 @@ void cbParseSql(void* param, TAOS_RES* res, int code) { char sql[128] = ""; sprintf(sql, "select last_row(*) from %s;", pStream->dstTable); taos_query_a(pSql->pTscObj, sql, fpStreamLastRow, param); - return ; } -TAOS_STREAM *taos_open_stream_withname(TAOS *taos, const char* dstTable, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), +TAOS_STREAM *taos_open_stream_withname(TAOS *taos, const char* dstTable, const char *sqlstr, void (*fp)(void *, TAOS_RES *, TAOS_ROW), int64_t stime, void *param, void (*callback)(void *), void* cqhandle) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) return NULL; @@ -697,14 +700,12 @@ TAOS_STREAM *taos_open_stream_withname(TAOS *taos, const char* dstTable, const c pStream->param = param; pStream->pSql = pSql; pStream->cqhandle = cqhandle; - pSql->pStream = pStream; - pSql->param = pStream; - pSql->maxRetry = TSDB_MAX_REPLICA; tscSetStreamDestTable(pStream, dstTable); pSql->pStream = pStream; pSql->param = pStream; pSql->maxRetry = TSDB_MAX_REPLICA; + pSql->sqlstr = calloc(1, strlen(sqlstr) + 1); if (pSql->sqlstr == NULL) { tscError("0x%"PRIx64" failed to malloc sql string buffer", pSql->self); @@ -725,14 +726,13 @@ TAOS_STREAM *taos_open_stream_withname(TAOS *taos, const char* dstTable, const c pSql->fp = cbParseSql; pSql->fetchFp = cbParseSql; - registerSqlObj(pSql); int32_t code = tsParseSql(pSql, true); if (code == TSDB_CODE_SUCCESS) { cbParseSql(pStream, pSql, code); } else if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - tscDebug(" CQ taso_open_stream IN Process. sql=%s", sqlstr); + tscDebug("0x%"PRIx64" CQ taso_open_stream IN Process", pSql->self); } else { tscError("0x%"PRIx64" open stream failed, sql:%s, code:%s", pSql->self, sqlstr, tstrerror(code)); taosReleaseRef(tscObjRef, pSql->self); @@ -743,7 +743,7 @@ TAOS_STREAM *taos_open_stream_withname(TAOS *taos, const char* dstTable, const c return pStream; } -TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), +TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *, TAOS_RES *, TAOS_ROW), int64_t stime, void *param, void (*callback)(void *)) { return taos_open_stream_withname(taos, "", sqlstr, fp, stime, param, callback, NULL); } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index eb7907f556..e1cb79b357 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2718,17 +2718,10 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO int32_t code = pParentSql->res.code; if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) && pParentSql->retry < pParentSql->maxRetry) { // remove the cached tableMeta and vgroup id list, and then parse the sql again - SSqlCmd* pParentCmd = &pParentSql->cmd; - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pParentCmd, 0); - tscRemoveTableMetaBuf(pTableMetaInfo, pParentSql->self); - tscResetSqlCmd(pParentCmd, true); + tscResetSqlCmd( &pParentSql->cmd, true, pParentSql->self); -// pParentCmd->pTableMetaMap = tscCleanupTableMetaMap(pParentCmd->pTableMetaMap); -// pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - - pParentSql->res.code = TSDB_CODE_SUCCESS; pParentSql->retry++; - + pParentSql->res.code = TSDB_CODE_SUCCESS; tscDebug("0x%"PRIx64" retry parse sql and send query, prev error: %s, retry:%d", pParentSql->self, tstrerror(code), pParentSql->retry); @@ -3143,7 +3136,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) numOfFailed += 1; // clean up tableMeta in cache - tscFreeQueryInfo(&pSql->cmd, false); + tscFreeQueryInfo(&pSql->cmd, false, pSql->self); SQueryInfo* pQueryInfo = tscGetQueryInfoS(&pSql->cmd); STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pParentObj->cmd, 0); tscAddTableMetaInfo(pQueryInfo, &pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); @@ -3165,7 +3158,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) } pParentObj->res.code = TSDB_CODE_SUCCESS; - tscResetSqlCmd(&pParentObj->cmd, false); + tscResetSqlCmd(&pParentObj->cmd, false, pParentObj->self); // in case of insert, redo parsing the sql string and build new submit data block for two reasons: // 1. the table Id(tid & uid) may have been update, the submit block needs to be updated accordingly. diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index a0e3c83a2e..a6712778ba 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1300,12 +1300,13 @@ static void tscDestroyResPointerInfo(SSqlRes* pRes) { pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free } -void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta) { +void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeCachedMeta, uint64_t id) { if (pCmd == NULL) { return; } SQueryInfo* pQueryInfo = pCmd->pQueryInfo; + while(pQueryInfo != NULL) { SQueryInfo* p = pQueryInfo->sibling; @@ -1314,7 +1315,7 @@ void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta) { SQueryInfo* pUpQueryInfo = taosArrayGetP(pQueryInfo->pUpstream, i); freeQueryInfoImpl(pUpQueryInfo); - clearAllTableMetaInfo(pUpQueryInfo, removeMeta); + clearAllTableMetaInfo(pUpQueryInfo, removeCachedMeta, id); if (pUpQueryInfo->pQInfo != NULL) { qDestroyQueryInfo(pUpQueryInfo->pQInfo); pUpQueryInfo->pQInfo = NULL; @@ -1330,7 +1331,7 @@ void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta) { } freeQueryInfoImpl(pQueryInfo); - clearAllTableMetaInfo(pQueryInfo, removeMeta); + clearAllTableMetaInfo(pQueryInfo, removeCachedMeta, id); if (pQueryInfo->pQInfo != NULL) { qDestroyQueryInfo(pQueryInfo->pQInfo); @@ -1359,7 +1360,7 @@ void destroyTableNameList(SInsertStatementParam* pInsertParam) { tfree(pInsertParam->pTableNameList); } -void tscResetSqlCmd(SSqlCmd* pCmd, bool clearCachedMeta) { +void tscResetSqlCmd(SSqlCmd* pCmd, bool clearCachedMeta, uint64_t id) { pCmd->command = 0; pCmd->numOfCols = 0; pCmd->count = 0; @@ -1373,20 +1374,8 @@ void tscResetSqlCmd(SSqlCmd* pCmd, bool clearCachedMeta) { tfree(pCmd->insertParam.tagData.data); pCmd->insertParam.tagData.dataLen = 0; - tscFreeQueryInfo(pCmd, clearCachedMeta); + tscFreeQueryInfo(pCmd, clearCachedMeta, id); pCmd->pTableMetaMap = tscCleanupTableMetaMap(pCmd->pTableMetaMap); - -// if (pCmd->pTableMetaMap != NULL) { -// STableMetaVgroupInfo* p = taosHashIterate(pCmd->pTableMetaMap, NULL); -// while (p) { -// taosArrayDestroy(p->vgroupIdList); -// tfree(p->pTableMeta); -// p = taosHashIterate(pCmd->pTableMetaMap, p); -// } -// -// taosHashCleanup(pCmd->pTableMetaMap); -// pCmd->pTableMetaMap = NULL; -// } } void* tscCleanupTableMetaMap(SHashObj* pTableMetaMap) { @@ -1501,7 +1490,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { pSql->self = 0; tscFreeSqlResult(pSql); - tscResetSqlCmd(pCmd, false); + tscResetSqlCmd(pCmd, false, pSql->self); memset(pCmd->payload, 0, (size_t)pCmd->allocSize); tfree(pCmd->payload); @@ -3369,20 +3358,15 @@ SArray* tscVgroupTableInfoDup(SArray* pVgroupTables) { return pa; } -void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta) { +void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta, uint64_t id) { for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - if (removeMeta) { - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + tscRemoveCachedTableMeta(pTableMetaInfo, id); } tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); tscClearTableMetaInfo(pTableMetaInfo); - - free(pTableMetaInfo); } tfree(pQueryInfo->pTableMetaInfo); @@ -3449,10 +3433,12 @@ void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo) { } tfree(pTableMetaInfo->pTableMeta); - pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); + tscColumnListDestroy(pTableMetaInfo->tagColList); pTableMetaInfo->tagColList = NULL; + + free(pTableMetaInfo); } void tscResetForNextRetrieve(SSqlRes* pRes) { @@ -3845,14 +3831,7 @@ static void tscSubqueryCompleteCallback(void* param, TAOS_RES* tres, int code) { // todo refactor tscDebug("0x%"PRIx64" all subquery response received, retry", pParentSql->self); - - SSqlCmd* pParentCmd = &pParentSql->cmd; - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pParentCmd, 0); - tscRemoveTableMetaBuf(pTableMetaInfo, pParentSql->self); - tscResetSqlCmd(pParentCmd, true); - -// pParentCmd->pTableMetaMap = tscCleanupTableMetaMap(pParentCmd->pTableMetaMap); -// pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + tscResetSqlCmd(&pParentSql->cmd, true, pParentSql->self); pParentSql->res.code = TSDB_CODE_SUCCESS; pParentSql->retry++; @@ -3871,7 +3850,7 @@ static void tscSubqueryCompleteCallback(void* param, TAOS_RES* tres, int code) { return; } - SQueryInfo *pQueryInfo = tscGetQueryInfo(pParentCmd); + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pParentSql->cmd); executeQuery(pParentSql, pQueryInfo); return; } @@ -4995,7 +4974,7 @@ SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg) { return info; } -void tscRemoveTableMetaBuf(STableMetaInfo* pTableMetaInfo, uint64_t id) { +void tscRemoveCachedTableMeta(STableMetaInfo* pTableMetaInfo, uint64_t id) { char fname[TSDB_TABLE_FNAME_LEN] = {0}; tNameExtractFullName(&pTableMetaInfo->name, fname); From cb9fedfb13f7b3ecadd32b4d168fa0105b50db32 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 10 Aug 2021 11:49:41 +0800 Subject: [PATCH 23/75] [TD-5931]:add empty table --- src/tsdb/src/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 716f82d154..795d446f4d 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -670,7 +670,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr SArray* px = taosArrayInit(4, sizeof(STableKeyInfo)); for (int32_t j = 0; j < numOfTables; ++j) { STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(oneGroup, j); - if (window->skey <= pInfo->lastKey && ((STable*)pInfo->pTable)->lastKey != TSKEY_INITIAL_VAL) { + if (window->skey <= pInfo->lastKey) { taosArrayPush(px, pInfo); pNew->numOfTables += 1; break; From e5ceaa2a9051e3002b81e14746bc2deded6e0b48 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 10 Aug 2021 14:37:19 +0800 Subject: [PATCH 24/75] [TD-5931]:invalidate time range when no tables after trimTableGroup --- src/tsdb/src/tsdbRead.c | 14 +++++++++++++- tests/script/general/parser/interp.sim | 13 ++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 795d446f4d..4735efceb8 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -670,7 +670,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr SArray* px = taosArrayInit(4, sizeof(STableKeyInfo)); for (int32_t j = 0; j < numOfTables; ++j) { STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(oneGroup, j); - if (window->skey <= pInfo->lastKey) { + if (window->skey <= pInfo->lastKey && ((STable*)pInfo->pTable)->lastKey != TSKEY_INITIAL_VAL) { taosArrayPush(px, pInfo); pNew->numOfTables += 1; break; @@ -691,6 +691,18 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pRef) { STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList); + if (pNew->numOfTables == 0) { + tsdbDebug("update query time range to invalidate time window"); + + assert(taosArrayGetSize(pNew->pGroupList) == 0); + bool asc = ASCENDING_TRAVERSE(pCond->order); + if (asc) { + pCond->twindow.ekey = pCond->twindow.skey - 1; + } else { + pCond->twindow.skey = pCond->twindow.ekey - 1; + } + } + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, pNew, qId, pRef); pQueryHandle->loadExternalRow = true; pQueryHandle->currentLoadExternalRows = true; diff --git a/tests/script/general/parser/interp.sim b/tests/script/general/parser/interp.sim index 3fb91e36c6..2dd3204e57 100644 --- a/tests/script/general/parser/interp.sim +++ b/tests/script/general/parser/interp.sim @@ -65,4 +65,15 @@ print ================== server restart completed run general/parser/interp_test.sim -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +print ================= TD-5931 +sql create stable st5931(ts timestamp, f int) tags(t int) +sql create table ct5931 using st5931 tags(1) +sql create table nt5931(ts timestamp, f int) +sql select interp(*) from nt5931 where ts=now +sql select interp(*) from st5931 where ts=now +sql select interp(*) from ct5931 where ts=now +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 570ecb9e079311519dd1cfc45a2a51b0696d1aed Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 10 Aug 2021 19:47:24 +0800 Subject: [PATCH 25/75] [TD-5838] fix tcp test error --- src/util/src/tnettest.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index 0bab7b7e66..4c180f9bbb 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -466,6 +466,7 @@ static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) { sendpkgLen = pkgLen; } + tsRpcForceTcp = 1; int32_t ret = taosNetCheckRpc(host, port, sendpkgLen, spi, NULL); if (ret < 0) { printf("failed to test TCP port:%d\n", port); @@ -479,6 +480,7 @@ static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) { sendpkgLen = pkgLen; } + tsRpcForceTcp = 0; ret = taosNetCheckRpc(host, port, pkgLen, spi, NULL); if (ret < 0) { printf("failed to test UDP port:%d\n", port); From 6b0fd5b2a35e5ba1299c771123874bb69dd405e9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 11 Aug 2021 13:54:19 +0800 Subject: [PATCH 26/75] [TD-5838] add function of speed test --- src/kit/shell/inc/shell.h | 2 + src/kit/shell/src/shellLinux.c | 13 +++++ src/kit/shell/src/shellMain.c | 4 +- src/kit/shell/src/shellWindows.c | 20 +++++++ src/util/inc/tnettest.h | 2 +- src/util/src/tnettest.c | 96 ++++++++++++++++++++++++++++++-- 6 files changed, 131 insertions(+), 6 deletions(-) diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h index 2374150c52..e19a18c9b7 100644 --- a/src/kit/shell/inc/shell.h +++ b/src/kit/shell/inc/shell.h @@ -56,6 +56,8 @@ typedef struct SShellArguments { int abort; int port; int pktLen; + int pktNum； + char* pktType, char* netTestRole; } SShellArguments; diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index dc74f6fcaa..fcba3a4426 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -50,6 +50,8 @@ static struct argp_option options[] = { {"timezone", 't', "TIMEZONE", 0, "Time zone of the shell, default is local."}, {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync."}, {"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."}, + {"pktnum", 'N', "PKTNUM", 0, "Packet numbers used for net test, default is 100."}, + {"pkttype", 'S', "PKTTYPE", 0, "Packet type used for net test, default is TCP."}, {0}}; static error_t parse_opt(int key, char *arg, struct argp_state *state) { @@ -148,6 +150,17 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { return -1; } break; + case 'N': + if (arg) { + arguments->pktNum = atoi(arg); + } else { + fprintf(stderr, "Invalid packet number\n"); + return -1; + } + break; + case 'S': + arguments->pktType = arg; + break; case OPT_ABORT: arguments->abort = 1; break; diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 0c70386061..eb6e2effc2 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -83,6 +83,8 @@ SShellArguments args = { .threadNum = 5, .commands = NULL, .pktLen = 1000, + .pktNum = 100, + .pktType = "TCP", .netTestRole = NULL }; @@ -116,7 +118,7 @@ int main(int argc, char* argv[]) { printf("Failed to init taos"); exit(EXIT_FAILURE); } - taosNetTest(args.netTestRole, args.host, args.port, args.pktLen); + taosNetTest(args.netTestRole, args.host, args.port, args.pktLen, args.pktLen, args.pktType); exit(0); } diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index 87d11a3516..ce161d89b7 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -55,6 +55,10 @@ void printHelp() { printf("%s%s%s\n", indent, indent, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync."); printf("%s%s\n", indent, "-l"); printf("%s%s%s\n", indent, indent, "Packet length used for net test, default is 1000 bytes."); + printf("%s%s\n", indent, "-N"); + printf("%s%s%s\n", indent, indent, "Packet numbers used for net test, default is 100."); + printf("%s%s\n", indent, "-S"); + printf("%s%s%s\n", indent, indent, "Packet type used for net test, default is TCP."); printf("%s%s\n", indent, "-V"); printf("%s%s%s\n", indent, indent, "Print program version."); @@ -170,6 +174,22 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { exit(EXIT_FAILURE); } } + else if (strcmp(argv[i], "-N") == 0) { + if (i < argc - 1) { + arguments->pktNum = atoi(argv[++i]); + } else { + fprintf(stderr, "option -N requires an argument\n"); + exit(EXIT_FAILURE); + } + } + else if (strcmp(argv[i], "-S") == 0) { + if (i < argc - 1) { + arguments->pktType = argv[++i]; + } else { + fprintf(stderr, "option -S requires an argument\n"); + exit(EXIT_FAILURE); + } + } else if (strcmp(argv[i], "-V") == 0) { printVersion(); exit(EXIT_SUCCESS); diff --git a/src/util/inc/tnettest.h b/src/util/inc/tnettest.h index b7585bd715..8a03b67628 100644 --- a/src/util/inc/tnettest.h +++ b/src/util/inc/tnettest.h @@ -20,7 +20,7 @@ extern "C" { #endif -void taosNetTest(char *role, char *host, int port, int pkgLen); +void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType); #ifdef __cplusplus } diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index 4c180f9bbb..fccdf1b24e 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -27,6 +27,10 @@ #include "syncMsg.h" #define MAX_PKG_LEN (64 * 1000) +#define MAX_SPEED_PKG_LEN (1024 * 1024) +#define MIN_SPEED_PKG_LEN 1024 +#define MAX_SPEED_PKG_NUM 10000 +#define MIN_SPEED_PKG_NUM 10 #define BUFFER_SIZE (MAX_PKG_LEN + 1024) extern int32_t tsRpcMaxUdpSize; @@ -544,12 +548,93 @@ static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) { } } -void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen) { +static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, + int32_t pkgNum, char *pkgType) { + char spi = 0; + + uInfo("check spend, host:%s port:%d pkgLen:%d pkgNum:%d pkgType:%s\\n", host, port, pkgLen, pkgNum, pkgType); + + SRpcEpSet epSet; + SRpcMsg reqMsg; + SRpcMsg rspMsg; + void * pRpcConn; + + char secretEncrypt[32] = {0}; + + pRpcConn = taosNetInitRpc(secretEncrypt, spi); + if (NULL == pRpcConn) { + uError("failed to init client rpc"); + return; + } + + memset(&epSet, 0, sizeof(SRpcEpSet)); + epSet.inUse = 0; + epSet.numOfEps = 1; + epSet.port[0] = port; + strcpy(epSet.fqdn[0], host); + + reqMsg.msgType = TSDB_MSG_TYPE_NETWORK_TEST; + reqMsg.pCont = rpcMallocCont(pkgLen); + reqMsg.contLen = pkgLen; + reqMsg.code = 0; + reqMsg.handle = NULL; // rpc handle returned to app + reqMsg.ahandle = NULL; // app handle set by client + strcpy(reqMsg.pCont, "nettest speed"); + + // record config + int32_t compressTmp = tsCompressMsgSize; + int32_t maxUdpSize = tsRpcMaxUdpSize; + + tsCompressMsgSize = -1; + if (0 == strcmp("TCP", pkgType)){ + tsRpcMaxUdpSize = 0; // force tcp + } else { + tsRpcMaxUdpSize = INT_MAX; + } + + int32_t totalSucc = 0; + int64_t startT = taosGetTimestampMs(); + for (int32_t i = 0; i < pkgNum; i++) { + int64_t startTime = taosGetTimestampMs(); + rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg); + + int code = 0; + if ((rspMsg.code != 0) || (rspMsg.msgType != TSDB_MSG_TYPE_NETWORK_TEST + 1)) { + uError("ret code 0x%x %s", rspMsg.code, tstrerror(rspMsg.code)); + code = -1; + }else{ + totalSucc ++; + } + + int64_t endTime = taosGetTimestampMs(); + int32_t el = endTime - startTime; + printf("progress: %5d/%d, status: %d, cost: %10d ms, speed: %10.2lf KB/s\n", i, pkgNum, code, el, pkgLen/(el/1000.0)/1024); + } + int64_t endT = taosGetTimestampMs(); + int32_t elT = endT - startT; + printf("total: %5d/%d, cost: %10d ms, speed: %10.2lf KB/s\n", totalSucc, pkgNum, elT, pkgLen * totalSucc/(elT/1000.0)/1024); + rpcFreeCont(rspMsg.pCont); + rpcClose(pRpcConn); + + // return config + tsCompressMsgSize = compressTmp; + tsRpcMaxUdpSize = maxUdpSize; +} + +void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, + int32_t pkgNum, char *pkgType) { tscEmbedded = 1; if (host == NULL) host = tsLocalFqdn; if (port == 0) port = tsServerPort; - if (pkgLen <= 10) pkgLen = 1000; - if (pkgLen > MAX_PKG_LEN) pkgLen = MAX_PKG_LEN; + if (0 == strcmp("speed", role){ + if (pkgLen <= MIN_SPEED_PKG_LEN) pkgLen = MIN_SPEED_PKG_LEN; + if (pkgLen > MAX_SPEED_PKG_LEN) pkgLen = MAX_SPEED_PKG_LEN; + if (pkgNum <= MIN_SPEED_PKG_NUM) pkgLen = MIN_SPEED_PKG_NUM; + if (pkgNum > MAX_SPEED_PKG_NUM) pkgLen = MAX_SPEED_PKG_NUM; + }else{ + if (pkgLen <= 10) pkgLen = 1000; + if (pkgLen > MAX_PKG_LEN) pkgLen = MAX_PKG_LEN; + } if (0 == strcmp("client", role)) { taosNetTestClient(host, port, pkgLen); @@ -562,7 +647,10 @@ void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen) { taosNetCheckSync(host, port); } else if (0 == strcmp("startup", role)) { taosNetTestStartup(host, port); - } else { + } else if (0 == strcmp("speed", role)) { + tscEmbedded = 0; + taosNetTestSpeed(host, port, pkgLen, pkgNum, pkgType); + }else { taosNetTestStartup(host, port); } From 2f91838e8eabc1d28e4373a670f2b5596aab5717 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 11 Aug 2021 13:58:32 +0800 Subject: [PATCH 27/75] [TD-5838] add function of speed test --- src/util/src/tnettest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index fccdf1b24e..6519f56121 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -626,7 +626,7 @@ void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, tscEmbedded = 1; if (host == NULL) host = tsLocalFqdn; if (port == 0) port = tsServerPort; - if (0 == strcmp("speed", role){ + if (0 == strcmp("speed", role)){ if (pkgLen <= MIN_SPEED_PKG_LEN) pkgLen = MIN_SPEED_PKG_LEN; if (pkgLen > MAX_SPEED_PKG_LEN) pkgLen = MAX_SPEED_PKG_LEN; if (pkgNum <= MIN_SPEED_PKG_NUM) pkgLen = MIN_SPEED_PKG_NUM; From d7311c0eaa8efcb135ede7f8b852fee06e1ba028 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 11 Aug 2021 13:59:58 +0800 Subject: [PATCH 28/75] [TD-5838] add function of speed test --- src/kit/shell/inc/shell.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h index e19a18c9b7..4d72d36e2e 100644 --- a/src/kit/shell/inc/shell.h +++ b/src/kit/shell/inc/shell.h @@ -56,8 +56,8 @@ typedef struct SShellArguments { int abort; int port; int pktLen; - int pktNum； - char* pktType, + int pktNum; + char* pktType; char* netTestRole; } SShellArguments; From 03063121dfcffcf7157455750223a065b7e40400 Mon Sep 17 00:00:00 2001 From: wpan Date: Wed, 11 Aug 2021 14:04:07 +0800 Subject: [PATCH 29/75] improve performance --- src/query/inc/qFilter.h | 15 +++- src/query/src/qFilter.c | 165 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 174 insertions(+), 6 deletions(-) diff --git a/src/query/inc/qFilter.h b/src/query/inc/qFilter.h index 7a7b3157ea..38da341c59 100644 --- a/src/query/inc/qFilter.h +++ b/src/query/inc/qFilter.h @@ -35,6 +35,8 @@ extern "C" { #define MAX_NUM_STR_SIZE 40 +#define FILTER_RM_UNIT_MIN_ROWS 1024 + enum { FLD_TYPE_COLUMN = 1, FLD_TYPE_VALUE = 2, @@ -70,6 +72,12 @@ enum { FI_STATUS_CLONED = 8, }; +enum { + FI_STATUS_BLK_ALL = 1, + FI_STATUS_BLK_EMPTY = 2, + FI_STATUS_BLK_ACTIVE = 4, +}; + enum { RANGE_TYPE_UNIT = 1, RANGE_TYPE_VAR_HASH = 2, @@ -197,6 +205,7 @@ typedef struct SFilterComUnit { void *colData; void *valData; void *valData2; + uint16_t colId; uint16_t dataSize; uint8_t dataType; uint8_t optr; @@ -224,7 +233,11 @@ typedef struct SFilterInfo { uint8_t *unitRes; // result uint8_t *unitFlags; // got result SFilterRangeCtx **colRange; - filter_exec_func func; + filter_exec_func func; + uint8_t blkFlag; + uint16_t blkGroupNum; + uint16_t *blkUnits; + int8_t *blkUnitRes; SFilterPCtx pctx; } SFilterInfo; diff --git a/src/query/src/qFilter.c b/src/query/src/qFilter.c index 1171bb0896..f4aec03513 100644 --- a/src/query/src/qFilter.c +++ b/src/query/src/qFilter.c @@ -157,7 +157,7 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val compareDoubleVal, compareLenPrefixedStr, compareStrPatternComp, compareFindItemInSet, compareWStrPatternComp, compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val, setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8 -}; +}; int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { int8_t comparFn = 0; @@ -2540,6 +2540,153 @@ int32_t filterUpdateComUnits(SFilterInfo *info) { return TSDB_CODE_SUCCESS; } +int32_t filterRmUnitByRange(SFilterInfo *info, SDataStatis *pDataStatis, int32_t numOfCols, int32_t numOfRows) { + int32_t rmUnit = 0; + + memset(info->blkUnitRes, 0, sizeof(*info->blkUnitRes) * info->unitNum); + + for (int32_t k = 0; k < info->unitNum; ++k) { + int32_t index = -1; + SFilterComUnit *cunit = &info->cunits[k]; + + if (FILTER_NO_MERGE_DATA_TYPE(cunit->dataType)) { + continue; + } + + for(int32_t i = 0; i < numOfCols; ++i) { + if (pDataStatis[i].colId == cunit->colId) { + index = i; + break; + } + } + + if (index == -1) { + continue; + } + + if (pDataStatis[index].numOfNull <= 0) { + if (cunit->optr == TSDB_RELATION_ISNULL) { + info->blkUnitRes[k] = -1; + rmUnit = 1; + continue; + } + + if (cunit->optr == TSDB_RELATION_NOTNULL) { + info->blkUnitRes[k] = 1; + rmUnit = 1; + continue; + } + } else { + if (pDataStatis[index].numOfNull == numOfRows) { + if (cunit->optr == TSDB_RELATION_ISNULL) { + info->blkUnitRes[k] = 1; + rmUnit = 1; + continue; + } + + info->blkUnitRes[k] = -1; + rmUnit = 1; + continue; + } + } + + if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL) { + continue; + } + + SDataStatis* pDataBlockst = &pDataStatis[index]; + void *minVal, *maxVal; + + if (cunit->dataType == TSDB_DATA_TYPE_FLOAT) { + float minv = (float)(*(double *)(&pDataBlockst->min)); + float maxv = (float)(*(double *)(&pDataBlockst->max)); + + minVal = &minv; + maxVal = &maxv; + } else { + minVal = &pDataBlockst->min; + maxVal = &pDataBlockst->max; + } + + bool minRes = false, maxRes = false; + + if (cunit->rfunc >= 0) { + minRes = (*gRangeCompare[cunit->rfunc])(minVal, minVal, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); + maxRes = (*gRangeCompare[cunit->rfunc])(maxVal, maxVal, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); + } else { + minRes = filterDoCompare(gDataCompare[cunit->func], cunit->optr, minVal, cunit->valData); + maxRes = filterDoCompare(gDataCompare[cunit->func], cunit->optr, maxVal, cunit->valData); + } + + if (minRes && maxRes) { + info->blkUnitRes[k] = 1; + rmUnit = 1; + } else if ((!minRes) && (!maxRes)) { + info->blkUnitRes[k] = -1; + rmUnit = 1; + } + } + + CHK_RET(rmUnit == 0, TSDB_CODE_SUCCESS); + + info->blkGroupNum = info->groupNum; + + uint16_t *unitNum = info->blkUnits; + uint16_t *unitIdx = unitNum + 1; + int32_t all = 0, empty = 0; + + for (uint32_t g = 0; g < info->groupNum; ++g) { + SFilterGroup *group = &info->groups[g]; + *unitNum = group->unitNum; + all = 0; + empty = 0; + + for (uint32_t u = 0; u < group->unitNum; ++u) { + uint16_t uidx = group->unitIdxs[u]; + if (info->blkUnitRes[u] == 1) { + --(*unitNum); + all = 1; + continue; + } else if (info->blkUnitRes[u] == -1) { + *unitNum = 0; + empty = 1; + break; + } + + *(unitIdx++) = uidx; + } + + if (*unitNum == 0) { + --info->blkGroupNum; + assert(empty || all); + + if (empty) { + FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY); + } else { + FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL); + goto _return; + } + + continue; + } + + unitNum = unitIdx; + ++unitIdx; + } + + if (info->blkGroupNum) { + FILTER_CLR_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY); + FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE); + } + +_return: + + qDebug("Block Filter Result:%s", FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY) ? "EMPTY" : (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL) ? "ALL" : "FILTER")); + + return TSDB_CODE_SUCCESS; +} + + static FORCE_INLINE bool filterExecuteImplAll(void *info, int32_t numOfRows, int8_t* p) { return true; @@ -2880,16 +3027,17 @@ bool filterRangeExecute(SFilterInfo *info, SDataStatis *pDataStatis, int32_t num // no statistics data, load the true data block if (index == -1) { - return true; + break; } // not support pre-filter operation on binary/nchar data type if (FILTER_NO_MERGE_DATA_TYPE(ctx->type)) { - return true; + break; } if ((pDataStatis[index].numOfNull <= 0) && (ctx->isnull && !ctx->notnull && !ctx->isrange)) { - return false; + ret = false; + break; } // all data in current column are NULL, no need to check its boundary value @@ -2897,7 +3045,8 @@ bool filterRangeExecute(SFilterInfo *info, SDataStatis *pDataStatis, int32_t num // if isNULL query exists, load the null data column if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) { - return false; + ret = false; + break; } continue; @@ -2929,6 +3078,12 @@ bool filterRangeExecute(SFilterInfo *info, SDataStatis *pDataStatis, int32_t num CHK_RET(!ret, ret); } + info->blkFlag = 0; + + if (ret && numOfRows >= FILTER_RM_UNIT_MIN_ROWS) { + filterRmUnitByRange(info, pDataStatis, numOfCols, numOfRows); + } + return ret; } From c388d7b818881a48dbc418863b389f16d44ac6c3 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 11 Aug 2021 17:16:26 +0800 Subject: [PATCH 30/75] [TD-5838] add function of speed test --- src/util/src/tnettest.c | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index 6519f56121..c946d87ff6 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -567,20 +567,6 @@ static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, return; } - memset(&epSet, 0, sizeof(SRpcEpSet)); - epSet.inUse = 0; - epSet.numOfEps = 1; - epSet.port[0] = port; - strcpy(epSet.fqdn[0], host); - - reqMsg.msgType = TSDB_MSG_TYPE_NETWORK_TEST; - reqMsg.pCont = rpcMallocCont(pkgLen); - reqMsg.contLen = pkgLen; - reqMsg.code = 0; - reqMsg.handle = NULL; // rpc handle returned to app - reqMsg.ahandle = NULL; // app handle set by client - strcpy(reqMsg.pCont, "nettest speed"); - // record config int32_t compressTmp = tsCompressMsgSize; int32_t maxUdpSize = tsRpcMaxUdpSize; @@ -596,6 +582,21 @@ static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, int64_t startT = taosGetTimestampMs(); for (int32_t i = 0; i < pkgNum; i++) { int64_t startTime = taosGetTimestampMs(); + + memset(&epSet, 0, sizeof(SRpcEpSet)); + epSet.inUse = 0; + epSet.numOfEps = 1; + epSet.port[0] = port; + strcpy(epSet.fqdn[0], host); + + reqMsg.msgType = TSDB_MSG_TYPE_NETWORK_TEST; + reqMsg.pCont = rpcMallocCont(pkgLen); + reqMsg.contLen = pkgLen; + reqMsg.code = 0; + reqMsg.handle = NULL; // rpc handle returned to app + reqMsg.ahandle = NULL; // app handle set by client + strcpy(reqMsg.pCont, "nettest speed"); + rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg); int code = 0; @@ -606,6 +607,8 @@ static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, totalSucc ++; } + rpcFreeCont(rspMsg.pCont); + int64_t endTime = taosGetTimestampMs(); int32_t el = endTime - startTime; printf("progress: %5d/%d, status: %d, cost: %10d ms, speed: %10.2lf KB/s\n", i, pkgNum, code, el, pkgLen/(el/1000.0)/1024); @@ -613,7 +616,7 @@ static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, int64_t endT = taosGetTimestampMs(); int32_t elT = endT - startT; printf("total: %5d/%d, cost: %10d ms, speed: %10.2lf KB/s\n", totalSucc, pkgNum, elT, pkgLen * totalSucc/(elT/1000.0)/1024); - rpcFreeCont(rspMsg.pCont); + rpcClose(pRpcConn); // return config From 684d9d911cbee77fcdf1e6d33596341cd5bb68c4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 11 Aug 2021 17:23:26 +0800 Subject: [PATCH 31/75] [TD-5838] add function of speed test --- src/util/src/tnettest.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index c946d87ff6..68a432ab79 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -27,10 +27,10 @@ #include "syncMsg.h" #define MAX_PKG_LEN (64 * 1000) -#define MAX_SPEED_PKG_LEN (1024 * 1024) +#define MAX_SPEED_PKG_LEN (1024 * 1024 * 1024) #define MIN_SPEED_PKG_LEN 1024 #define MAX_SPEED_PKG_NUM 10000 -#define MIN_SPEED_PKG_NUM 10 +#define MIN_SPEED_PKG_NUM 1 #define BUFFER_SIZE (MAX_PKG_LEN + 1024) extern int32_t tsRpcMaxUdpSize; @@ -579,9 +579,9 @@ static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, } int32_t totalSucc = 0; - int64_t startT = taosGetTimestampMs(); - for (int32_t i = 0; i < pkgNum; i++) { - int64_t startTime = taosGetTimestampMs(); + int64_t startT = taosGetTimestampUs(); + for (int32_t i = 1; i <= pkgNum; i++) { + int64_t startTime = taosGetTimestampUs(); memset(&epSet, 0, sizeof(SRpcEpSet)); epSet.inUse = 0; @@ -609,11 +609,11 @@ static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, rpcFreeCont(rspMsg.pCont); - int64_t endTime = taosGetTimestampMs(); + int64_t endTime = taosGetTimestampUs(); int32_t el = endTime - startTime; printf("progress: %5d/%d, status: %d, cost: %10d ms, speed: %10.2lf KB/s\n", i, pkgNum, code, el, pkgLen/(el/1000.0)/1024); } - int64_t endT = taosGetTimestampMs(); + int64_t endT = taosGetTimestampUs(); int32_t elT = endT - startT; printf("total: %5d/%d, cost: %10d ms, speed: %10.2lf KB/s\n", totalSucc, pkgNum, elT, pkgLen * totalSucc/(elT/1000.0)/1024); @@ -632,8 +632,8 @@ void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, if (0 == strcmp("speed", role)){ if (pkgLen <= MIN_SPEED_PKG_LEN) pkgLen = MIN_SPEED_PKG_LEN; if (pkgLen > MAX_SPEED_PKG_LEN) pkgLen = MAX_SPEED_PKG_LEN; - if (pkgNum <= MIN_SPEED_PKG_NUM) pkgLen = MIN_SPEED_PKG_NUM; - if (pkgNum > MAX_SPEED_PKG_NUM) pkgLen = MAX_SPEED_PKG_NUM; + if (pkgNum <= MIN_SPEED_PKG_NUM) pkgNum = MIN_SPEED_PKG_NUM; + if (pkgNum > MAX_SPEED_PKG_NUM) pkgNum = MAX_SPEED_PKG_NUM; }else{ if (pkgLen <= 10) pkgLen = 1000; if (pkgLen > MAX_PKG_LEN) pkgLen = MAX_PKG_LEN; From 4ff07fe4152629c5a6eedf2d9f1b6908a66255a9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 11 Aug 2021 17:32:01 +0800 Subject: [PATCH 32/75] [TD-5838] add function of speed test --- src/kit/shell/src/shellMain.c | 2 +- src/util/src/tnettest.c | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index eb6e2effc2..496dac8524 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -118,7 +118,7 @@ int main(int argc, char* argv[]) { printf("Failed to init taos"); exit(EXIT_FAILURE); } - taosNetTest(args.netTestRole, args.host, args.port, args.pktLen, args.pktLen, args.pktType); + taosNetTest(args.netTestRole, args.host, args.port, args.pktLen, args.pktNum, args.pktType); exit(0); } diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index 68a432ab79..b88ab7e6e6 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -552,7 +552,7 @@ static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType) { char spi = 0; - uInfo("check spend, host:%s port:%d pkgLen:%d pkgNum:%d pkgType:%s\\n", host, port, pkgLen, pkgNum, pkgType); + uInfo("check spend, host:%s port:%d pkgLen:%d pkgNum:%d pkgType:%s", host, port, pkgLen, pkgNum, pkgType); SRpcEpSet epSet; SRpcMsg reqMsg; @@ -579,9 +579,9 @@ static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, } int32_t totalSucc = 0; - int64_t startT = taosGetTimestampUs(); + uint64_t startT = taosGetTimestampUs(); for (int32_t i = 1; i <= pkgNum; i++) { - int64_t startTime = taosGetTimestampUs(); + uint64_t startTime = taosGetTimestampUs(); memset(&epSet, 0, sizeof(SRpcEpSet)); epSet.inUse = 0; @@ -609,13 +609,13 @@ static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, rpcFreeCont(rspMsg.pCont); - int64_t endTime = taosGetTimestampUs(); - int32_t el = endTime - startTime; - printf("progress: %5d/%d, status: %d, cost: %10d ms, speed: %10.2lf KB/s\n", i, pkgNum, code, el, pkgLen/(el/1000.0)/1024); + uint64_t endTime = taosGetTimestampUs(); + uint64_t el = endTime - startTime; + printf("progress: %5d/%d, status: %d, cost: %8" PRId64 " ms, speed: %8.2lf KB/s\n", i, pkgNum, code, el, pkgLen/(el/1000000.0)/1024); } int64_t endT = taosGetTimestampUs(); - int32_t elT = endT - startT; - printf("total: %5d/%d, cost: %10d ms, speed: %10.2lf KB/s\n", totalSucc, pkgNum, elT, pkgLen * totalSucc/(elT/1000.0)/1024); + uint64_t elT = endT - startT; + printf("total: %5d/%d, cost: %8" PRId64 " ms, speed: %8.2lf KB/s\n", totalSucc, pkgNum, elT, pkgLen * totalSucc/(elT/1000000.0)/1024); rpcClose(pRpcConn); From a40d817ac063eda80b83dcf7e8a6a93aeacfaf1b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 11 Aug 2021 17:41:00 +0800 Subject: [PATCH 33/75] [TD-5838] add function of speed test --- src/util/src/tnettest.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index b88ab7e6e6..914750f5f8 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -611,11 +611,11 @@ static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, uint64_t endTime = taosGetTimestampUs(); uint64_t el = endTime - startTime; - printf("progress: %5d/%d, status: %d, cost: %8" PRId64 " ms, speed: %8.2lf KB/s\n", i, pkgNum, code, el, pkgLen/(el/1000000.0)/1024); + printf("progress: %5d/%d, status: %d, cost: %10.2lf ms, speed: %10.2lf MB/s\n", i, pkgNum, code, el/1000.0, pkgLen/(el/1000000.0)/1024.0/1024.0); } int64_t endT = taosGetTimestampUs(); uint64_t elT = endT - startT; - printf("total: %5d/%d, cost: %8" PRId64 " ms, speed: %8.2lf KB/s\n", totalSucc, pkgNum, elT, pkgLen * totalSucc/(elT/1000000.0)/1024); + printf("total succ: %5d/%d, cost: %10.2lf ms, speed: %10.2lf MB/s\n", totalSucc, pkgNum, elT/1000.0, pkgLen * totalSucc/(elT/1000000.0)/1024.0/1024.0); rpcClose(pRpcConn); From 3b70249a2704ddd6c6228683c8322e66f03ad864 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 11 Aug 2021 17:42:45 +0800 Subject: [PATCH 34/75] [TD-5910]obtain coredump execfn in script --- tests/test-all.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test-all.sh b/tests/test-all.sh index 6e7963e787..c179811ce6 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -25,7 +25,7 @@ function stopTaosd { function dohavecore(){ corefile=`find $corepath -mmin 1` core_file=`echo $corefile|cut -d " " -f2` - proc=`echo $corefile|cut -d "_" -f3` + proc=`file $core_file|awk -F "execfn:" '/execfn:/{print $2}'|tr -d \' |awk '{print $1}'|tr -d \,` if [ -n "$corefile" ];then echo 'taosd or taos has generated core' rm case.log @@ -46,7 +46,7 @@ function dohavecore(){ fi fi if [[ $1 == 1 ]];then - echo '\n'|gdb /usr/local/taos/bin/$proc $core_file -ex "bt 10" -ex quit + echo '\n'|gdb $proc $core_file -ex "bt 10" -ex quit exit 8 fi fi From 842098942b22cbabbccd540fb4e6693bd244ac85 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 11 Aug 2021 17:48:56 +0800 Subject: [PATCH 35/75] [td-225]update the log --- src/client/src/tscServer.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 11c458e9bc..87289f7728 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -525,6 +525,7 @@ static void doProcessMsgFromServer(SSchedMsg* pSchedMsg) { } void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { + int64_t st = taosGetTimestampUs(); SSchedMsg schedMsg = {0}; schedMsg.fp = doProcessMsgFromServer; @@ -543,6 +544,11 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { schedMsg.msg = NULL; taosScheduleTask(tscQhandle, &schedMsg); + + int64_t et = taosGetTimestampUs(); + if (et - st > 100) { + tscDebug("add message to task queue, elapsed time:%"PRId64, et - st); + } } int doBuildAndSendMsg(SSqlObj *pSql) { From 8cb3fb045bbce8776883a6eee7760e8c623bee34 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 11 Aug 2021 17:57:20 +0800 Subject: [PATCH 36/75] [TD-5838] add function of speed test --- src/util/src/tnettest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index 914750f5f8..cf83ae61f0 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -615,7 +615,7 @@ static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, } int64_t endT = taosGetTimestampUs(); uint64_t elT = endT - startT; - printf("total succ: %5d/%d, cost: %10.2lf ms, speed: %10.2lf MB/s\n", totalSucc, pkgNum, elT/1000.0, pkgLen * totalSucc/(elT/1000000.0)/1024.0/1024.0); + printf("total succ: %5d/%d, cost: %10.2lf ms, speed: %10.2lf MB/s\n", totalSucc, pkgNum, elT/1000.0, pkgLen/(elT/1000000.0)/1024.0/1024.0*totalSucc); rpcClose(pRpcConn); From 198f93ed4ea389449e40680b48d1599d8b929f9e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 11 Aug 2021 18:02:36 +0800 Subject: [PATCH 37/75] [TD-5838] add function of speed test --- src/util/src/tnettest.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index cf83ae61f0..a44bbb3df4 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -611,11 +611,11 @@ static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, uint64_t endTime = taosGetTimestampUs(); uint64_t el = endTime - startTime; - printf("progress: %5d/%d, status: %d, cost: %10.2lf ms, speed: %10.2lf MB/s\n", i, pkgNum, code, el/1000.0, pkgLen/(el/1000000.0)/1024.0/1024.0); + printf("progress:%5d/%d\t\tstatus:%d\t\tcost:%8.2lf ms\t\tspeed:%8.2lf MB/s\n", i, pkgNum, code, el/1000.0, pkgLen/(el/1000000.0)/1024.0/1024.0); } int64_t endT = taosGetTimestampUs(); uint64_t elT = endT - startT; - printf("total succ: %5d/%d, cost: %10.2lf ms, speed: %10.2lf MB/s\n", totalSucc, pkgNum, elT/1000.0, pkgLen/(elT/1000000.0)/1024.0/1024.0*totalSucc); + printf("\ntotal succ:%5d/%d\t\tcost:%8.2lf ms\t\tspeed:%8.2lf MB/s\n", totalSucc, pkgNum, elT/1000.0, pkgLen/(elT/1000000.0)/1024.0/1024.0*totalSucc); rpcClose(pRpcConn); From 8017ce03b2c6333c008cda7f4896bed567017f53 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 11 Aug 2021 18:04:09 +0800 Subject: [PATCH 38/75] [TD-5838] add function of speed test --- src/util/src/tnettest.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index a44bbb3df4..bfbbfd3415 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -611,11 +611,11 @@ static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, uint64_t endTime = taosGetTimestampUs(); uint64_t el = endTime - startTime; - printf("progress:%5d/%d\t\tstatus:%d\t\tcost:%8.2lf ms\t\tspeed:%8.2lf MB/s\n", i, pkgNum, code, el/1000.0, pkgLen/(el/1000000.0)/1024.0/1024.0); + printf("progress:%5d/%d\tstatus:%d\tcost:%8.2lf ms\tspeed:%8.2lf MB/s\n", i, pkgNum, code, el/1000.0, pkgLen/(el/1000000.0)/1024.0/1024.0); } int64_t endT = taosGetTimestampUs(); uint64_t elT = endT - startT; - printf("\ntotal succ:%5d/%d\t\tcost:%8.2lf ms\t\tspeed:%8.2lf MB/s\n", totalSucc, pkgNum, elT/1000.0, pkgLen/(elT/1000000.0)/1024.0/1024.0*totalSucc); + printf("\ntotal succ:%5d/%d\tcost:%8.2lf ms\tspeed:%8.2lf MB/s\n", totalSucc, pkgNum, elT/1000.0, pkgLen/(elT/1000000.0)/1024.0/1024.0*totalSucc); rpcClose(pRpcConn); From 6cf5becd39d1a384a835b379d2b6143f8f3a30f3 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 11 Aug 2021 18:18:56 +0800 Subject: [PATCH 39/75] [TD-5838] add function of speed test --- src/util/src/tnettest.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index bfbbfd3415..8e6dd7e20e 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -570,11 +570,14 @@ static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, // record config int32_t compressTmp = tsCompressMsgSize; int32_t maxUdpSize = tsRpcMaxUdpSize; + int32_t forceTcp = tsRpcForceTcp; tsCompressMsgSize = -1; if (0 == strcmp("TCP", pkgType)){ + tsRpcForceTcp = 1; tsRpcMaxUdpSize = 0; // force tcp } else { + tsRpcForceTcp = 0; tsRpcMaxUdpSize = INT_MAX; } @@ -622,6 +625,7 @@ static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, // return config tsCompressMsgSize = compressTmp; tsRpcMaxUdpSize = maxUdpSize; + tsRpcForceTcp = forceTcp; } void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, From 6c8f6e153a98435cdfb5ce50bffc74ce8da6af5a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 11 Aug 2021 23:46:16 +0800 Subject: [PATCH 40/75] [TD-5838] add function of speed test --- src/util/src/tnettest.c | 63 ++++++++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 20 deletions(-) diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index 8e6dd7e20e..435c1e936f 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -548,39 +548,49 @@ static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) { } } -static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, - int32_t pkgNum, char *pkgType) { - char spi = 0; - - uInfo("check spend, host:%s port:%d pkgLen:%d pkgNum:%d pkgType:%s", host, port, pkgLen, pkgNum, pkgType); - - SRpcEpSet epSet; - SRpcMsg reqMsg; - SRpcMsg rspMsg; - void * pRpcConn; - - char secretEncrypt[32] = {0}; - - pRpcConn = taosNetInitRpc(secretEncrypt, spi); - if (NULL == pRpcConn) { - uError("failed to init client rpc"); - return; +static void taosNetTestFqdn(char *host) { + int code = 0; + uint64_t startTime = taosGetTimestampUs(); + uint32_t ip = taosGetIpv4FromFqdn(host); + if (ip == 0xffffffff) { + uError("failed to get IP address from %s since %s", host, strerror(errno)); + code = -1; } + uint64_t endTime = taosGetTimestampUs(); + uint64_t el = endTime - startTime; + printf("check convert fqdn spend, statuw: %d\tcost: %" PRIu64 " us\n", code, el); + return; +} +static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen, + int32_t pkgNum, char *pkgType) { // record config int32_t compressTmp = tsCompressMsgSize; int32_t maxUdpSize = tsRpcMaxUdpSize; int32_t forceTcp = tsRpcForceTcp; - tsCompressMsgSize = -1; - if (0 == strcmp("TCP", pkgType)){ + if (0 == strcmp("tcp", pkgType)){ tsRpcForceTcp = 1; tsRpcMaxUdpSize = 0; // force tcp } else { tsRpcForceTcp = 0; tsRpcMaxUdpSize = INT_MAX; } + tsCompressMsgSize = -1; + SRpcEpSet epSet; + SRpcMsg reqMsg; + SRpcMsg rspMsg; + void * pRpcConn; + char secretEncrypt[32] = {0}; + char spi = 0; + pRpcConn = taosNetInitRpc(secretEncrypt, spi); + if (NULL == pRpcConn) { + uError("failed to init client rpc"); + return; + } + + printf("check net spend, host:%s port:%d pkgLen:%d pkgNum:%d pkgType:%s\n", host, port, pkgLen, pkgNum, pkgType); int32_t totalSucc = 0; uint64_t startT = taosGetTimestampUs(); for (int32_t i = 1; i <= pkgNum; i++) { @@ -626,6 +636,18 @@ static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, tsCompressMsgSize = compressTmp; tsRpcMaxUdpSize = maxUdpSize; tsRpcForceTcp = forceTcp; + return; +} + +static void taosNetTestSpeed(char *host, int32_t port, int32_t pkgLen, + int32_t pkgNum, char *pkgType) { + if (0 == strcmp("fqdn", pkgType)){ + taosNetTestFqdn(host); + return; + } + + taosNetCheckSpeed(host, port, pkgLen, pkgNum, pkgType); + return; } void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, @@ -656,7 +678,8 @@ void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, taosNetTestStartup(host, port); } else if (0 == strcmp("speed", role)) { tscEmbedded = 0; - taosNetTestSpeed(host, port, pkgLen, pkgNum, pkgType); + char type[10] = {0}; + taosNetTestSpeed(host, port, pkgLen, pkgNum, strtolower(type, pkgType)); }else { taosNetTestStartup(host, port); } From 7ec7776cea6b6c6dfbb627ea82f74d419741bc98 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 Aug 2021 17:34:41 +0000 Subject: [PATCH 41/75] [TD-5610] resp detailed info of invalidfqdn --- src/client/src/tscServer.c | 19 ++++++++++++++++--- src/rpc/src/rpcMain.c | 4 ++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index f0ee180bbe..fde063b38f 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -502,13 +502,26 @@ static void doProcessMsgFromServer(SSchedMsg* pSchedMsg) { } rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code; if (pRes->code == TSDB_CODE_RPC_FQDN_ERROR) { + + tscAllocPayload(pCmd, TSDB_FQDN_LEN + 64); + // handle three situation + // 1. epset retry, only return last failure ep + // 2. no epset retry, like 'taos -h invalidFqdn', return invalidFqdn + // 3. other situation, not expected if (pEpSet) { - char buf[TSDB_FQDN_LEN + 64] = {0}; - tscAllocPayload(pCmd, sizeof(buf)); sprintf(tscGetErrorMsgPayload(pCmd), "%s\"%s\"", tstrerror(pRes->code),pEpSet->fqdn[(pEpSet->inUse)%(pEpSet->numOfEps)]); + } else if (pCmd->command >= TSDB_SQL_MGMT) { + SRpcEpSet tEpset; + + SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + taosCorBeginRead(&pCorEpSet->version); + tEpset = pCorEpSet->epSet; + taosCorEndRead(&pCorEpSet->version); + + sprintf(tscGetErrorMsgPayload(pCmd), "%s\"%s\"", tstrerror(pRes->code),tEpset.fqdn[(tEpset.inUse)%(tEpset.numOfEps)]); } else { sprintf(tscGetErrorMsgPayload(pCmd), "%s", tstrerror(pRes->code)); - } + } } (*pSql->fp)(pSql->param, pSql, rpcMsg->code); } diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index c93a3f929d..e958a8e5ec 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1133,8 +1133,8 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { } else { // for asynchronous API SRpcEpSet *pEpSet = NULL; - //if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect) - pEpSet = &pContext->epSet; + if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect) + pEpSet = &pContext->epSet; (*pRpc->cfp)(pMsg, pEpSet); } From 6b0dfd475e86096f9567e6c1163f41cbcd933a86 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 12 Aug 2021 09:42:57 +0800 Subject: [PATCH 42/75] [TD-5838] add function of speed test --- src/util/src/tnettest.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index 435c1e936f..153ad346db 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -558,7 +558,7 @@ static void taosNetTestFqdn(char *host) { } uint64_t endTime = taosGetTimestampUs(); uint64_t el = endTime - startTime; - printf("check convert fqdn spend, statuw: %d\tcost: %" PRIu64 " us\n", code, el); + printf("check convert fqdn spend, status: %d\tcost: %" PRIu64 " us\n", code, el); return; } @@ -590,7 +590,7 @@ static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen, return; } - printf("check net spend, host:%s port:%d pkgLen:%d pkgNum:%d pkgType:%s\n", host, port, pkgLen, pkgNum, pkgType); + printf("check net spend, host:%s port:%d pkgLen:%d pkgNum:%d pkgType:%s\n\n", host, port, pkgLen, pkgNum, pkgType); int32_t totalSucc = 0; uint64_t startT = taosGetTimestampUs(); for (int32_t i = 1; i <= pkgNum; i++) { From 7a924a11e739cffecc56ebbac6d6d2cdb206cf0f Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 12 Aug 2021 11:29:18 +0800 Subject: [PATCH 43/75] add some log --- tests/test-all.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test-all.sh b/tests/test-all.sh index c179811ce6..1753da3d36 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -26,6 +26,7 @@ function dohavecore(){ corefile=`find $corepath -mmin 1` core_file=`echo $corefile|cut -d " " -f2` proc=`file $core_file|awk -F "execfn:" '/execfn:/{print $2}'|tr -d \' |awk '{print $1}'|tr -d \,` + echo $core_file,$corefile if [ -n "$corefile" ];then echo 'taosd or taos has generated core' rm case.log From f7d04ae567acc15e5a772a7b733700b5720eac24 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 Aug 2021 04:05:57 +0000 Subject: [PATCH 44/75] [TD-5797] function forbidden --- src/client/src/tscSQLParser.c | 26 ++++++++++++++++++++++++-- src/client/src/tscUtil.c | 1 - 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index a6db366643..fe5b6990ed 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2043,8 +2043,12 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS const char* msg1 = "too many items in selection clause"; const char* msg2 = "functions or others can not be mixed up"; const char* msg3 = "not support query expression"; - const char* msg4 = "only support distinct one column or tag"; + const char* msg4 = "invalid distinct query"; const char* msg5 = "invalid function name"; + const char* msg6 = "not support distinct mixed with agg function"; + const char* msg7 = "not support distinct mixed with join"; + const char* msg8 = "not support distinct mixed with groupby"; + const char* msg9 = "not support distinct in nest query"; // too many result columns not support order by in query if (taosArrayGetSize(pSelNodeList) > TSDB_MAX_COLUMNS) { @@ -2056,6 +2060,7 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS } bool hasDistinct = false; + bool hasAgg = false; size_t numOfExpr = taosArrayGetSize(pSelNodeList); for (int32_t i = 0; i < numOfExpr; ++i) { int32_t outputIndex = (int32_t)tscNumOfExprs(pQueryInfo); @@ -2067,6 +2072,7 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS int32_t type = pItem->pNode->type; if (type == SQL_NODE_SQLFUNCTION) { + hasAgg = true; pItem->pNode->functionId = isValidFunction(pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n); SUdfInfo* pUdfInfo = NULL; if (pItem->pNode->functionId < 0) { @@ -2102,10 +2108,25 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS } } + + //TODO(dengyihao), refactor as function + //handle distinct func mixed with other func if (hasDistinct == true) { if (!isValidDistinctSql(pQueryInfo) ) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } + if (hasAgg) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); + } + if (joinQuery) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); + } + if (pQueryInfo->groupbyExpr.numOfGroupCols != 0) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); + } + if (pQueryInfo->pDownstream != NULL) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); + } pQueryInfo->distinct = true; } @@ -8395,12 +8416,12 @@ static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pS pSub->pUdfInfo = pUdfInfo; pSub->udfCopy = true; + pSub->pDownstream = pQueryInfo; int32_t code = validateSqlNode(pSql, p, pSub); if (code != TSDB_CODE_SUCCESS) { return code; } - pSub->pDownstream = pQueryInfo; // create dummy table meta info STableMetaInfo* pTableMetaInfo1 = calloc(1, sizeof(STableMetaInfo)); @@ -8509,6 +8530,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf if (validateGroupbyNode(pQueryInfo, pSqlNode->pGroupby, pCmd) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } + if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, false, timeWindowQuery, true) != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 5a724af7bf..008c5c0a43 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1502,7 +1502,6 @@ void tscFreeSqlObj(SSqlObj* pSql) { tscFreeSqlResult(pSql); tscResetSqlCmd(pCmd, false); - memset(pCmd->payload, 0, (size_t)pCmd->allocSize); tfree(pCmd->payload); pCmd->allocSize = 0; From a75137581e0c0a4b52d1f22eb1e24a26607d7cc8 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 12 Aug 2021 13:42:22 +0800 Subject: [PATCH 45/75] update case --- tests/test-all.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test-all.sh b/tests/test-all.sh index 1753da3d36..8dd1ade9be 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -24,10 +24,9 @@ function stopTaosd { function dohavecore(){ corefile=`find $corepath -mmin 1` - core_file=`echo $corefile|cut -d " " -f2` - proc=`file $core_file|awk -F "execfn:" '/execfn:/{print $2}'|tr -d \' |awk '{print $1}'|tr -d \,` - echo $core_file,$corefile if [ -n "$corefile" ];then + core_file=`echo $corefile|cut -d " " -f2` + proc=`file $core_file|awk -F "execfn:" '/execfn:/{print $2}'|tr -d \' |awk '{print $1}'|tr -d \,` echo 'taosd or taos has generated core' rm case.log if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]] && [[ $1 == 1 ]]; then From b89aef4479fb01fb65d84e59617fd1294500234d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 Aug 2021 05:43:06 +0000 Subject: [PATCH 46/75] [TD-5610] make jenkins happy --- src/client/src/tscServer.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index fde063b38f..2f891af1a7 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -502,12 +502,11 @@ static void doProcessMsgFromServer(SSchedMsg* pSchedMsg) { } rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code; if (pRes->code == TSDB_CODE_RPC_FQDN_ERROR) { - tscAllocPayload(pCmd, TSDB_FQDN_LEN + 64); // handle three situation // 1. epset retry, only return last failure ep // 2. no epset retry, like 'taos -h invalidFqdn', return invalidFqdn - // 3. other situation, not expected + // 3. other situation, no expected if (pEpSet) { sprintf(tscGetErrorMsgPayload(pCmd), "%s\"%s\"", tstrerror(pRes->code),pEpSet->fqdn[(pEpSet->inUse)%(pEpSet->numOfEps)]); } else if (pCmd->command >= TSDB_SQL_MGMT) { From d82e04be8c273c7af37d32683c20d0545e69ee3e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 12 Aug 2021 13:50:29 +0800 Subject: [PATCH 47/75] [td-5881]: Sort the result according to any single column in the outer query result is allowed. --- src/client/inc/tscUtil.h | 11 ++--- src/client/src/tscSQLParser.c | 85 ++++++++++++++++++++--------------- src/client/src/tscServer.c | 6 +-- src/client/src/tscUtil.c | 2 - src/query/inc/qExecutor.h | 2 +- src/query/src/qExecutor.c | 64 +++++++++++++++----------- src/query/src/qPlan.c | 15 ++++--- 7 files changed, 106 insertions(+), 79 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index b74fe29a61..17e05b1111 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -29,15 +29,16 @@ extern "C" { #include "tsched.h" #include "tsclient.h" -#define UTIL_TABLE_IS_SUPER_TABLE(metaInfo) \ +#define UTIL_TABLE_IS_SUPER_TABLE(metaInfo) \ (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE)) + #define UTIL_TABLE_IS_CHILD_TABLE(metaInfo) \ (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE)) - -#define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo)\ - (!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo))) -#define UTIL_TABLE_IS_TMP_TABLE(metaInfo) \ +#define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo) \ + (!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo) || UTIL_TABLE_IS_TMP_TABLE(metaInfo))) + +#define UTIL_TABLE_IS_TMP_TABLE(metaInfo) \ (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_TEMP_TABLE)) #pragma pack(push,1) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index d6df43af68..4e51d4a294 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5747,14 +5747,19 @@ static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { pQueryInfo->order.order = TSDB_ORDER_ASC; if (isTopBottomQuery(pQueryInfo)) { pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; - } else { // in case of select tbname from super_table, the defualt order column can not be the primary ts column - pQueryInfo->order.orderColId = INT32_MIN; + } else { // in case of select tbname from super_table, the default order column can not be the primary ts column + pQueryInfo->order.orderColId = INT32_MIN; // todo define a macro } /* for super table query, set default ascending order for group output */ if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; } + + if (pQueryInfo->distinct) { + pQueryInfo->order.order = TSDB_ORDER_ASC; + pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; + } } int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, SSchema* pSchema) { @@ -5766,17 +5771,12 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq setDefaultOrderInfo(pQueryInfo); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - if (pQueryInfo->distinct == true) { - pQueryInfo->order.order = TSDB_ORDER_ASC; - pQueryInfo->order.orderColId = 0; - return TSDB_CODE_SUCCESS; - } - if (pSqlNode->pSortOrder == NULL) { + if (pQueryInfo->distinct || pSqlNode->pSortOrder == NULL) { return TSDB_CODE_SUCCESS; } - SArray* pSortorder = pSqlNode->pSortOrder; + char* pMsgBuf = tscGetErrorMsgPayload(pCmd); + SArray* pSortOrder = pSqlNode->pSortOrder; /* * for table query, there is only one or none order option is allowed, which is the @@ -5784,19 +5784,19 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq * * for super table query, the order option must be less than 3. */ - size_t size = taosArrayGetSize(pSortorder); - if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { + size_t size = taosArrayGetSize(pSortOrder); + if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_TMP_TABLE(pTableMetaInfo)) { if (size > 1) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); + return invalidOperationMsg(pMsgBuf, msg0); } } else { if (size > 2) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(pMsgBuf, msg3); } } // handle the first part of order by - tVariant* pVar = taosArrayGet(pSortorder, 0); + tVariant* pVar = taosArrayGet(pSortOrder, 0); // e.g., order by 1 asc, return directly with out further check. if (pVar->nType >= TSDB_DATA_TYPE_TINYINT && pVar->nType <= TSDB_DATA_TYPE_BIGINT) { @@ -5808,7 +5808,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // super table query if (getColumnIndexByName(&columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(pMsgBuf, msg1); } bool orderByTags = false; @@ -5820,7 +5820,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq // it is a tag column if (pQueryInfo->groupbyExpr.columnInfo == NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(pMsgBuf, msg2); } SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); if (relTagIndex == pColIndex->colIndex) { @@ -5841,13 +5841,14 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq orderByGroupbyCol = true; } } + if (!(orderByTags || orderByTS || orderByGroupbyCol) && !isTopBottomQuery(pQueryInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidOperationMsg(pMsgBuf, msg3); } else { // order by top/bottom result value column is not supported in case of interval query. assert(!(orderByTags && orderByTS && orderByGroupbyCol)); } - size_t s = taosArrayGetSize(pSortorder); + size_t s = taosArrayGetSize(pSortOrder); if (s == 1) { if (orderByTags) { pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); @@ -5866,7 +5867,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq pExpr = tscExprGet(pQueryInfo, 1); if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(pMsgBuf, msg2); } tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); @@ -5884,9 +5885,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq addPrimaryTsColIntoResult(pQueryInfo, pCmd); } } - } - - if (s == 2) { + } else { tVariantListItem *pItem = taosArrayGet(pSqlNode->pSortOrder, 0); if (orderByTags) { pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); @@ -5903,22 +5902,23 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq tVariant* pVar2 = &pItem->pVar; SStrToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz}; if (getColumnIndexByName(&cname, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidOperationMsg(pMsgBuf, msg1); } if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(pMsgBuf, msg2); } else { - tVariantListItem* p1 = taosArrayGet(pSortorder, 1); + tVariantListItem* p1 = taosArrayGet(pSortOrder, 1); pQueryInfo->order.order = p1->sortOrder; pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; } } - } else { // meter query - if (getColumnIndexByName(&columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); + } else if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo)) { // check order by clause for normal table & temp table + if (getColumnIndexByName(&columnName, pQueryInfo, &index, pMsgBuf) != TSDB_CODE_SUCCESS) { + return invalidOperationMsg(pMsgBuf, msg1); } + if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX && !isTopBottomQuery(pQueryInfo)) { bool validOrder = false; SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo; @@ -5926,13 +5926,14 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq SColIndex* pColIndex = taosArrayGet(columnInfo, 0); validOrder = (pColIndex->colIndex == index.columnIndex); } + if (!validOrder) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(pMsgBuf, msg2); } + tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); pQueryInfo->groupbyExpr.orderIndex = pSchema[index.columnIndex].colId; pQueryInfo->groupbyExpr.orderType = p1->sortOrder; - } if (isTopBottomQuery(pQueryInfo)) { @@ -5948,13 +5949,14 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq pExpr = tscExprGet(pQueryInfo, 1); if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(pMsgBuf, msg2); } + validOrder = true; } if (!validOrder) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidOperationMsg(pMsgBuf, msg2); } tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); @@ -5964,6 +5966,18 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq return TSDB_CODE_SUCCESS; } + tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); + pQueryInfo->order.order = pItem->sortOrder; + pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; + } else { + // handle the temp table order by clause. You can order by any single column in case of the temp table, created by + // inner subquery. + assert(UTIL_TABLE_IS_TMP_TABLE(pTableMetaInfo) && taosArrayGetSize(pSqlNode->pSortOrder) == 1); + + if (getColumnIndexByName(&columnName, pQueryInfo, &index, pMsgBuf) != TSDB_CODE_SUCCESS) { + return invalidOperationMsg(pMsgBuf, msg1); + } + tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); pQueryInfo->order.order = pItem->sortOrder; pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; @@ -8736,8 +8750,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf const char* msg8 = "condition missing for join query"; const char* msg9 = "not support 3 level select"; - int32_t code = TSDB_CODE_SUCCESS; - + int32_t code = TSDB_CODE_SUCCESS; SSqlCmd* pCmd = &pSql->cmd; STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -9028,8 +9041,6 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf pQueryInfo->simpleAgg = isSimpleAggregateRv(pQueryInfo); pQueryInfo->onlyTagQuery = onlyTagPrjFunction(pQueryInfo); pQueryInfo->groupbyColumn = tscGroupbyColumn(pQueryInfo); - //pQueryInfo->globalMerge = tscIsTwoStageSTableQuery(pQueryInfo, 0); - pQueryInfo->arithmeticOnAgg = tsIsArithmeticQueryOnAggResult(pQueryInfo); pQueryInfo->orderProjectQuery = tscOrderedProjectionQueryOnSTable(pQueryInfo, 0); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 87289f7728..dd8e95d664 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -903,16 +903,16 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; SQueryAttr query = {{0}}; tscCreateQueryFromQueryInfo(pQueryInfo, &query, pSql); + query.vgId = pTableMeta->vgId; SArray* tableScanOperator = createTableScanPlan(&query); SArray* queryOperator = createExecOperatorPlan(&query); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload; tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version)); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 4857265d17..b134b6c0c9 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1228,11 +1228,9 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue if (pCond && pCond->cond) { createQueryFilter(pCond->cond, pCond->len, &pFilters); } - //createInputDataFlterInfo(px, numOfCol1, &numOfFilterCols, &pFilterInfo); } SOperatorInfo* pSourceOperator = createDummyInputOperator(pSqlObjList[0], pSchema, numOfCol1, pFilters); - pOutput->precision = pSqlObjList[0]->res.precision; SSchema* schema = NULL; diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index a65ef524ae..a2947678d7 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -421,7 +421,6 @@ typedef struct STableScanInfo { int32_t *rowCellInfoOffset; SExprInfo *pExpr; SSDataBlock block; - bool loadExternalRows; // load external rows (prev & next rows) int32_t numOfOutput; int64_t elapsedTime; @@ -579,6 +578,7 @@ SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperator int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter); SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput); +SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 4d697b06a5..e8c4d57bfa 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -223,6 +223,7 @@ static void destroySFillOperatorInfo(void* param, int32_t numOfOutput); static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput); static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput); static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput); +static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); @@ -2036,6 +2037,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf } break; } + case OP_StateWindow: { pRuntimeEnv->proot = createStatewindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; @@ -2052,24 +2054,20 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf case OP_Filter: { // todo refactor int32_t numOfFilterCols = 0; -// if (pQueryAttr->numOfFilterCols > 0) { -// pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, -// pQueryAttr->numOfOutput, pQueryAttr->tableCols, pQueryAttr->numOfFilterCols); -// } else { - if (pQueryAttr->stableQuery) { - SColumnInfo* pColInfo = - extractColumnFilterInfo(pQueryAttr->pExpr3, pQueryAttr->numOfExpr3, &numOfFilterCols); - pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, - pQueryAttr->numOfExpr3, pColInfo, numOfFilterCols); - freeColumnInfo(pColInfo, pQueryAttr->numOfExpr3); - } else { - SColumnInfo* pColInfo = - extractColumnFilterInfo(pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &numOfFilterCols); - pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, - pQueryAttr->numOfOutput, pColInfo, numOfFilterCols); - freeColumnInfo(pColInfo, pQueryAttr->numOfOutput); - } -// } + if (pQueryAttr->stableQuery) { + SColumnInfo* pColInfo = + extractColumnFilterInfo(pQueryAttr->pExpr3, pQueryAttr->numOfExpr3, &numOfFilterCols); + pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, + pQueryAttr->numOfExpr3, pColInfo, numOfFilterCols); + freeColumnInfo(pColInfo, pQueryAttr->numOfExpr3); + } else { + SColumnInfo* pColInfo = + extractColumnFilterInfo(pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &numOfFilterCols); + pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, + pQueryAttr->numOfOutput, pColInfo, numOfFilterCols); + freeColumnInfo(pColInfo, pQueryAttr->numOfOutput); + } + break; } @@ -2081,11 +2079,12 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf case OP_MultiwayMergeSort: { bool groupMix = true; - if(pQueryAttr->slimit.offset != 0 || pQueryAttr->slimit.limit != -1) { + if (pQueryAttr->slimit.offset != 0 || pQueryAttr->slimit.limit != -1) { groupMix = false; } + pRuntimeEnv->proot = createMultiwaySortOperatorInfo(pRuntimeEnv, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, - 4096, merger, groupMix); // TODO hack it + 4096, merger, groupMix); // TODO hack it break; } @@ -2106,6 +2105,11 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf break; } + case OP_Order: { + pRuntimeEnv->proot = createOrderOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + break; + } + default: { assert(0); } @@ -5213,8 +5217,8 @@ static int32_t doMergeSDatablock(SSDataBlock* pDest, SSDataBlock* pSrc) { int32_t numOfCols = pSrc->info.numOfCols; for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock->pData, i); - SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock->pData, i); + SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); + SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); int32_t newSize = (pDest->info.rows + pSrc->info.rows) * pCol2->info.bytes; char* tmp = realloc(pCol2->pData, newSize); @@ -5227,6 +5231,7 @@ static int32_t doMergeSDatablock(SSDataBlock* pDest, SSDataBlock* pSrc) { } } + pDest->info.rows += pSrc->info.rows; return TSDB_CODE_SUCCESS; } @@ -5277,7 +5282,7 @@ static SSDataBlock* doSort(void* param, bool* newgroup) { return (pInfo->pDataBlock->info.rows > 0)? pInfo->pDataBlock:NULL; } -SOperatorInfo *createOrderOperatorInfo(SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo)); { @@ -5292,17 +5297,20 @@ SOperatorInfo *createOrderOperatorInfo(SExprInfo* pExpr, int32_t numOfOutput) { } pDataBlock->info.numOfCols = numOfOutput; + pInfo->pDataBlock = pDataBlock; } SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "Order"; + pOperator->name = "InMemoryOrder"; pOperator->operatorType = OP_Order; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->exec = doSort; - pOperator->cleanup = NULL; + pOperator->cleanup = destroyOrderOperatorInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + appendUpstream(pOperator, upstream); return pOperator; } @@ -6197,6 +6205,11 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { pInfo->pRes = destroyOutputBuf(pInfo->pRes); } +static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { + SOrderOperatorInfo* pInfo = (SOrderOperatorInfo*) param; + pInfo->pDataBlock = destroyOutputBuf(pInfo->pDataBlock); +} + static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) { SFilterOperatorInfo* pInfo = (SFilterOperatorInfo*) param; doDestroyFilterInfo(pInfo->pFilterInfo, pInfo->numOfFilterCols); @@ -6494,7 +6507,6 @@ SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorIn pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doFill; pOperator->cleanup = destroySFillOperatorInfo; diff --git a/src/query/src/qPlan.c b/src/query/src/qPlan.c index e01f41276f..1a4fe5b5c0 100644 --- a/src/query/src/qPlan.c +++ b/src/query/src/qPlan.c @@ -557,10 +557,9 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { int32_t op = 0; if (onlyQueryTags(pQueryAttr)) { // do nothing for tags query - if (onlyQueryTags(pQueryAttr)) { - op = OP_TagScan; - taosArrayPush(plan, &op); - } + op = OP_TagScan; + taosArrayPush(plan, &op); + if (pQueryAttr->distinct) { op = OP_Distinct; taosArrayPush(plan, &op); @@ -643,8 +642,14 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { taosArrayPush(plan, &op); } } + + // outer query order by support + int32_t orderColId = pQueryAttr->order.orderColId; + if (pQueryAttr->vgId == 0 && orderColId != PRIMARYKEY_TIMESTAMP_COL_INDEX && orderColId != INT32_MIN) { + op = OP_Order; + taosArrayPush(plan, &op); + } } - if (pQueryAttr->limit.limit > 0 || pQueryAttr->limit.offset > 0) { op = OP_Limit; From 255e3b0d646e21e8590d3d6f4920c6c6a9fec5c9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 12 Aug 2021 16:03:49 +0800 Subject: [PATCH 48/75] [td-5881]: Sort the result according to any single column in the outer query result is allowed. --- src/query/inc/qExecutor.h | 2 +- src/query/src/qExecutor.c | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index a2947678d7..eaa4da1ca8 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -578,7 +578,7 @@ SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperator int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter); SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput); -SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrderVal* pOrderVal); SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index e8c4d57bfa..477dfaa982 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2106,7 +2106,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf } case OP_Order: { - pRuntimeEnv->proot = createOrderOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + pRuntimeEnv->proot = createOrderOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &pQueryAttr->order); break; } @@ -5232,6 +5232,7 @@ static int32_t doMergeSDatablock(SSDataBlock* pDest, SSDataBlock* pSrc) { } pDest->info.rows += pSrc->info.rows; + return TSDB_CODE_SUCCESS; } @@ -5282,7 +5283,7 @@ static SSDataBlock* doSort(void* param, bool* newgroup) { return (pInfo->pDataBlock->info.rows > 0)? pInfo->pDataBlock:NULL; } -SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrderVal* pOrderVal) { SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo)); { @@ -5290,10 +5291,14 @@ SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI pDataBlock->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); for(int32_t i = 0; i < numOfOutput; ++i) { SColumnInfoData col = {{0}}; - col.info.bytes = pExpr->base.resBytes; - col.info.colId = pExpr->base.resColId; - col.info.type = pExpr->base.resType; + col.info.colId = pExpr[i].base.colInfo.colId; + col.info.bytes = pExpr[i].base.colBytes; + col.info.type = pExpr[i].base.colType; taosArrayPush(pDataBlock->pDataBlock, &col); + + if (col.info.colId == pOrderVal->orderColId) { + pInfo->colIndex = i; + } } pDataBlock->info.numOfCols = numOfOutput; From 789918b993c6c06b1ec95fb7701b034013d1b237 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 12 Aug 2021 16:21:48 +0800 Subject: [PATCH 49/75] [td-5881]fix bug in TD-5881 --- src/query/src/qExecutor.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 477dfaa982..abdd296c09 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5302,6 +5302,7 @@ SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI } pDataBlock->info.numOfCols = numOfOutput; + pInfo->order = pOrderVal->order; pInfo->pDataBlock = pDataBlock; } From ee3afee2be09e7bec5cc8a83e5ec03485c3a273a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 Aug 2021 08:23:24 +0000 Subject: [PATCH 50/75] [TD-5797] fix invalid table bug --- src/client/src/tscServer.c | 2 +- src/client/src/tscUtil.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 3e8dfac1da..752e60b3b0 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -409,7 +409,7 @@ static void doProcessMsgFromServer(SSchedMsg* pSchedMsg) { if ((TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY | TSDB_QUERY_TYPE_SUBQUERY | TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) && !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY)) || - (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY))) { + (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY)) || (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->distinct)) { // do nothing in case of super table subquery } else { pSql->retry += 1; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 008c5c0a43..6ef9cbae5c 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -3620,7 +3620,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t pNewQueryInfo->prjOffset = pQueryInfo->prjOffset; pNewQueryInfo->numOfTables = 0; pNewQueryInfo->pTableMetaInfo = NULL; - pNewQueryInfo->bufLen = pQueryInfo->bufLen; + pNewQueryInfo->bufLen = pQueryInfo->bufLen; + pNewQueryInfo->distinct = pQueryInfo->distinct; pNewQueryInfo->buf = malloc(pQueryInfo->bufLen); if (pNewQueryInfo->buf == NULL) { From 7abcd83812cee62cb71f594904c11a204586e4af Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 Aug 2021 12:03:05 +0000 Subject: [PATCH 51/75] [TD-5797] fix invalid table bug --- src/client/src/tscSQLParser.c | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index fe5b6990ed..90ad811e55 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1940,20 +1940,6 @@ static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) { pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; } -bool isValidDistinctSql(SQueryInfo* pQueryInfo) { - if (pQueryInfo == NULL) { - return false; - } - if ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_QUERY) != TSDB_QUERY_TYPE_STABLE_QUERY - && (pQueryInfo->type & TSDB_QUERY_TYPE_TABLE_QUERY) != TSDB_QUERY_TYPE_TABLE_QUERY) { - return false; - } - //if (tscNumOfExprs(pQueryInfo) == 1){ - // return true; - //} - return true; -} - static bool hasNoneUserDefineExpr(SQueryInfo* pQueryInfo) { size_t numOfExprs = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < numOfExprs; ++i) { @@ -2043,7 +2029,7 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS const char* msg1 = "too many items in selection clause"; const char* msg2 = "functions or others can not be mixed up"; const char* msg3 = "not support query expression"; - const char* msg4 = "invalid distinct query"; + const char* msg4 = "not support distinct mixed with proj"; const char* msg5 = "invalid function name"; const char* msg6 = "not support distinct mixed with agg function"; const char* msg7 = "not support distinct mixed with join"; @@ -2062,12 +2048,14 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS bool hasDistinct = false; bool hasAgg = false; size_t numOfExpr = taosArrayGetSize(pSelNodeList); + int32_t distIdx = -1; for (int32_t i = 0; i < numOfExpr; ++i) { int32_t outputIndex = (int32_t)tscNumOfExprs(pQueryInfo); tSqlExprItem* pItem = taosArrayGet(pSelNodeList, i); if (hasDistinct == false) { hasDistinct = (pItem->distinct == true); + distIdx = i; } int32_t type = pItem->pNode->type; @@ -2112,9 +2100,9 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS //TODO(dengyihao), refactor as function //handle distinct func mixed with other func if (hasDistinct == true) { - if (!isValidDistinctSql(pQueryInfo) ) { + if (distIdx != 0) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - } + } if (hasAgg) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } From 708216e6930513ffa51c8bc4c416f3a5109fad41 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Thu, 12 Aug 2021 21:59:52 +0800 Subject: [PATCH 52/75] [TD-5722]test testing one column distinct with subtable --- tests/pytest/fulltest.sh | 1 + tests/pytest/query/distinctOneColTb.py | 280 +++++++++++++++++++++++++ 2 files changed, 281 insertions(+) create mode 100644 tests/pytest/query/distinctOneColTb.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 137069e6b6..0f4c7a806d 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -212,6 +212,7 @@ python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestQueryWithJson.py #query +python3 test.py -f query/distinctOneColTb.py python3 ./test.py -f query/filter.py python3 ./test.py -f query/filterCombo.py python3 ./test.py -f query/queryNormal.py diff --git a/tests/pytest/query/distinctOneColTb.py b/tests/pytest/query/distinctOneColTb.py new file mode 100644 index 0000000000..1f6089629a --- /dev/null +++ b/tests/pytest/query/distinctOneColTb.py @@ -0,0 +1,280 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def tb_all_query(self, num, sql="tb_all", where=""): + tdSql.query( + f"select distinct(ts2) from {sql} {where}") + tdSql.checkRows(num) + tdSql.query( + f"select distinct(cint) from {sql} {where}") + tdSql.checkRows(num) + tdSql.query( + f"select distinct(cbigint) from {sql} {where}") + tdSql.checkRows(num) + tdSql.query( + f"select distinct(csmallint) from {sql} {where}") + tdSql.checkRows(num) + tdSql.query( + f"select distinct(ctinyint) from {sql} {where}") + tdSql.checkRows(num) + tdSql.query( + f"select distinct(cfloat) from {sql} {where}") + tdSql.checkRows(num) + tdSql.query( + f"select distinct(cdouble) from {sql} {where}") + tdSql.checkRows(num) + tdSql.query( + f"select distinct(cbool) from {sql} {where}") + if num < 2: + tdSql.checkRows(num) + else: + tdSql.checkRows(2) + tdSql.query( + f"select distinct(cbinary) from {sql} {where}") + tdSql.checkRows(num) + tdSql.query( + f"select distinct(cnchar) from {sql} {where}") + tdSql.checkRows(num) + + tdSql.query( + f"select distinct(ts2) as a from {sql} {where}") + tdSql.checkRows(num) + tdSql.query( + f"select distinct(cint) as a from {sql} {where}") + tdSql.checkRows(num) + tdSql.query( + f"select distinct(cbigint) as a from {sql} {where}") + tdSql.checkRows(num) + tdSql.query( + f"select distinct(csmallint) as a from {sql} {where}") + tdSql.checkRows(num) + tdSql.query( + f"select distinct(ctinyint) as a from {sql} {where}") + tdSql.checkRows(num) + tdSql.query( + f"select distinct(cfloat) as a from {sql} {where}") + tdSql.checkRows(num) + tdSql.query( + f"select distinct(cdouble) as a from {sql} {where}") + tdSql.checkRows(num) + tdSql.query( + f"select distinct(cbool) as a from {sql} {where}") + if num < 2: + tdSql.checkRows(num) + else: + tdSql.checkRows(2) + tdSql.query( + f"select distinct(cbinary) as a from {sql} {where}") + tdSql.checkRows(num) + tdSql.query( + f"select distinct(cnchar) as a from {sql} {where}") + tdSql.checkRows(num) + + + def tb_all_query_sub(self, num, sql="tb_all", where="",colName = "c1"): + tdSql.query( + f"select distinct({colName}) from {sql} {where}") + tdSql.checkRows(num) + + def errorCheck(self, sql='tb_all'): + tdSql.error(f"select distinct() from {sql}") + tdSql.error(f"select distinct from {sql}") + tdSql.error(f"distinct ts2 from {sql}") + tdSql.error(f"distinct(ts2) from {sql}") + tdSql.error(f"distinct c1 from") + tdSql.error(f"distinct(c1) from") + tdSql.error(f"select distinct(ts2), avg(cint) from {sql}") + tdSql.error(f"select distinct ts2, avg(cint) from {sql}") + + ## the following line is going to core dump + #tdSql.error(f"select avg(cint), distinct(ts2) from {sql}") + #tdSql.error(f"select avg(cint), distinct ts2 from {sql}") + + tdSql.error(f"select distinct(ts2), cint from {sql}") + tdSql.error(f"select distinct ts2, cint from {sql}") + tdSql.error(f"select distinct(ts2) from {sql} group by cint") + tdSql.error(f"select distinct ts2 from {sql} group by cint") + tdSql.error(f"select distinct(ts2) from {sql} intervasl(1s)") + tdSql.error(f"select distinct ts2 from {sql} interval(1s)") + tdSql.error(f"select distinct(ts2) from {sql} slimit 1 soffset 1") + tdSql.error(f"select distinct ts2 from {sql} slimit 1 soffset 1") + tdSql.error(f"select distinct(ts2) from {sql} slimit 1") + tdSql.error(f"select distinct ts2 from {sql} slimit 1") + + ##order by is not supported but not being prohibited + #tdSql.error(f"select distinct(ts2) from {sql} order by desc") + #tdSql.error(f"select distinct ts2 from {sql} order by desc") + #tdSql.error(f"select distinct(ts2) from {sql} order by asc") + #tdSql.error(f"select distinct ts2 from {sql} order by asc") + + ##distinct should not use on first ts, but it can be applied + #tdSql.error(f"select distinct(ts) from {sql}") + #tdSql.error(f"select distinct ts from {sql}") + + ##distinct should not be used in inner query, but error did not occur + # tdSql.error(f"select distinct(ts2) from (select distinct ts2 from {sql})") + # tdSql.error(f"select distinct ts2 from (select distinct ts2 from {sql})") + + + def query_all_tb(self, whereNum=0,inNum=0,maxNum=0,tbName="tb_all"): + self.tb_all_query(maxNum,sql=tbName) + + self.tb_all_query(num=whereNum,where="where ts2 = '2021-08-06 18:01:50.550'",sql=tbName) + self.tb_all_query(num=whereNum,where="where cint = 1073741820",sql=tbName) + self.tb_all_query(num=whereNum,where="where cbigint = 1125899906842620",sql=tbName) + self.tb_all_query(num=whereNum,where="where csmallint = 150",sql=tbName) + self.tb_all_query(num=whereNum,where="where ctinyint = 10",sql=tbName) + self.tb_all_query(num=whereNum,where="where cfloat = 0.10",sql=tbName) + self.tb_all_query(num=whereNum,where="where cdouble = 0.130",sql=tbName) + self.tb_all_query(num=whereNum,where="where cbool = true",sql=tbName) + self.tb_all_query(num=whereNum,where="where cbinary = 'adc1'",sql=tbName) + self.tb_all_query(num=whereNum,where="where cnchar = '双精度浮'",sql=tbName) + self.tb_all_query(num=inNum,where="where ts2 in ( '2021-08-06 18:01:50.550' , '2021-08-06 18:01:50.551' )",sql=tbName) + self.tb_all_query(num=inNum,where="where cint in (1073741820,1073741821)",sql=tbName) + self.tb_all_query(num=inNum,where="where cbigint in (1125899906842620,1125899906842621)",sql=tbName) + self.tb_all_query(num=inNum,where="where csmallint in (150,151)",sql=tbName) + self.tb_all_query(num=inNum,where="where ctinyint in (10,11)",sql=tbName) + self.tb_all_query(num=inNum,where="where cfloat in (0.10,0.11)",sql=tbName) + self.tb_all_query(num=inNum,where="where cdouble in (0.130,0.131)",sql=tbName) + self.tb_all_query(num=inNum,where="where cbinary in ('adc0','adc1')",sql=tbName) + self.tb_all_query(num=inNum,where="where cnchar in ('双','双精度浮')",sql=tbName) + self.tb_all_query(num=inNum,where="where cbool in (0, 1)",sql=tbName) + self.tb_all_query(num=whereNum,where="where cnchar like '双__浮'",sql=tbName) + self.tb_all_query(num=maxNum,where="where cnchar like '双%'",sql=tbName) + self.tb_all_query(num=maxNum,where="where cbinary like 'adc_'",sql=tbName) + self.tb_all_query(num=maxNum,where="where cbinary like 'a%'",sql=tbName) + self.tb_all_query(num=whereNum,where="limit 1",sql=tbName) + self.tb_all_query(num=whereNum,where="limit 1 offset 1",sql=tbName) + + #subquery + self.tb_all_query(num=maxNum,sql=f'(select * from {tbName})') + + #subquery with where in outer query + self.tb_all_query(num=whereNum,where="where ts2 = '2021-08-06 18:01:50.550'",sql=f'(select * from {tbName})') + self.tb_all_query(num=whereNum,where="where cint = 1073741820",sql=f'(select * from {tbName})') + self.tb_all_query(num=whereNum,where="where cbigint = 1125899906842620",sql=f'(select * from {tbName})') + self.tb_all_query(num=whereNum,where="where csmallint = 150",sql=f'(select * from {tbName})') + self.tb_all_query(num=whereNum,where="where ctinyint = 10",sql=f'(select * from {tbName})') + self.tb_all_query(num=whereNum,where="where cfloat = 0.10",sql=f'(select * from {tbName})') + self.tb_all_query(num=whereNum,where="where cdouble = 0.130",sql=f'(select * from {tbName})') + self.tb_all_query(num=whereNum,where="where cbool = true",sql=f'(select * from {tbName})') + self.tb_all_query(num=whereNum,where="where cbinary = 'adc1'",sql=f'(select * from {tbName})') + self.tb_all_query(num=whereNum,where="where cnchar = '双精度浮'",sql=f'(select * from {tbName})') + self.tb_all_query(num=inNum,where="where ts2 in ( '2021-08-06 18:01:50.550' , '2021-08-06 18:01:50.551' )",sql=f'(select * from {tbName})') + self.tb_all_query(num=inNum,where="where cint in (1073741820,1073741821)",sql=f'(select * from {tbName})') + self.tb_all_query(num=inNum,where="where cbigint in (1125899906842620,1125899906842621)",sql=f'(select * from {tbName})') + self.tb_all_query(num=inNum,where="where csmallint in (150,151)",sql=f'(select * from {tbName})') + self.tb_all_query(num=inNum,where="where ctinyint in (10,11)",sql=f'(select * from {tbName})') + self.tb_all_query(num=inNum,where="where cfloat in (0.10,0.11)",sql=f'(select * from {tbName})') + self.tb_all_query(num=inNum,where="where cdouble in (0.130,0.131)",sql=f'(select * from {tbName})') + self.tb_all_query(num=inNum,where="where cbinary in ('adc0','adc1')",sql=f'(select * from {tbName})') + self.tb_all_query(num=inNum,where="where cnchar in ('双','双精度浮')",sql=f'(select * from {tbName})') + self.tb_all_query(num=inNum,where="where cbool in (0, 1)",sql=f'(select * from {tbName})') + self.tb_all_query(num=whereNum,where="where cnchar like '双__浮'",sql=f'(select * from {tbName})') + self.tb_all_query(num=maxNum,where="where cnchar like '双%'",sql=f'(select * from {tbName})') + self.tb_all_query(num=maxNum,where="where cbinary like 'adc_'",sql=f'(select * from {tbName})') + self.tb_all_query(num=maxNum,where="where cbinary like 'a%'",sql=f'(select * from {tbName})') + self.tb_all_query(num=whereNum,where="limit 1",sql=f'(select * from {tbName})') + self.tb_all_query(num=whereNum,where="limit 1 offset 1",sql=f'(select * from {tbName})') + #table query with inner query has error + tdSql.error('select distinct ts2 from (select )') + #table query with error option + self.errorCheck() + + + def run(self): + tdSql.prepare() + + tdLog.notice( + "==============phase1 distinct(col1) with no values==========") + tdSql.execute("create stable if not exists stb_all (ts timestamp, ts2 timestamp, cint int, cbigint bigint, csmallint smallint, ctinyint tinyint,cfloat float, cdouble double, cbool bool, cbinary binary(32), cnchar nchar(32)) tags(tint int)") + tdSql.execute( + "create table if not exists tb_all using stb_all tags(1)") + self.query_all_tb() + + tdLog.notice( + "==============phase1 finished ==========\n\n\n") + + tdLog.notice( + "==============phase2 distinct(col1) all values are null==========") + tdSql.execute( + "insert into tb_all values(now,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)") + tdSql.execute( + "insert into tb_all values(now,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)") + tdSql.execute( + "insert into tb_all values(now,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)") + + # normal query + self.query_all_tb() + + tdLog.notice( + "==============phase2 finished ==========\n\n\n") + + + tdLog.notice( + "==============phase3 distinct with distinct values ==========\n\n\n") + tdSql.execute( + "insert into tb_all values(now+1s,'2021-08-06 18:01:50.550',1073741820,1125899906842620,150,10,0.10,0.130, true,'adc0','双')") + tdSql.execute( + "insert into tb_all values(now+2s,'2021-08-06 18:01:50.551',1073741821,1125899906842621,151,11,0.11,0.131,false,'adc1','双精度浮')") + tdSql.execute( + "insert into tb_all values(now+3s,'2021-08-06 18:01:50.552',1073741822,1125899906842622,152,12,0.12,0.132,NULL,'adc2','双精度')") + tdSql.execute( + "insert into tb_all values(now+4s,'2021-08-06 18:01:50.553',1073741823,1125899906842623,153,13,0.13,0.133,NULL,'adc3','双精')") + tdSql.execute( + "insert into tb_all values(now+5s,'2021-08-06 18:01:50.554',1073741824,1125899906842624,154,14,0.14,0.134,NULL,'adc4','双精度浮点型')") + + # normal query + self.query_all_tb(maxNum=5,inNum=2,whereNum=1) + + tdLog.notice( + "==============phase3 finishes ==========\n\n\n") + + tdLog.notice( + "==============phase4 distinct with some values the same values ==========\n\n\n") + tdSql.execute( + "insert into tb_all values(now+10s,'2021-08-06 18:01:50.550',1073741820,1125899906842620,150,10,0.10,0.130, true,'adc0','双')") + tdSql.execute( + "insert into tb_all values(now+20s,'2021-08-06 18:01:50.551',1073741821,1125899906842621,151,11,0.11,0.131,false,'adc1','双精度浮')") + tdSql.execute( + "insert into tb_all values(now+30s,'2021-08-06 18:01:50.552',1073741822,1125899906842622,152,12,0.12,0.132,NULL,'adc2','双精度')") + tdSql.execute( + "insert into tb_all values(now+40s,'2021-08-06 18:01:50.553',1073741823,1125899906842623,153,13,0.13,0.133,NULL,'adc3','双精')") + tdSql.execute( + "insert into tb_all values(now+50s,'2021-08-06 18:01:50.554',1073741824,1125899906842624,154,14,0.14,0.134,NULL,'adc4','双精度浮点型')") + # normal query + self.query_all_tb(maxNum=5,inNum=2,whereNum=1) + + tdLog.notice( + "==============phase4 finishes ==========\n\n\n") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 649b740c18d8ae33a00c930acb337ee762327d0e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 Aug 2021 18:19:38 +0000 Subject: [PATCH 53/75] [TD-5797] refactor errmsg --- src/client/src/tscSQLParser.c | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 90ad811e55..277c9a527b 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2029,12 +2029,11 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS const char* msg1 = "too many items in selection clause"; const char* msg2 = "functions or others can not be mixed up"; const char* msg3 = "not support query expression"; - const char* msg4 = "not support distinct mixed with proj"; + const char* msg4 = "not support distinct mixed with proj/agg func"; const char* msg5 = "invalid function name"; - const char* msg6 = "not support distinct mixed with agg function"; - const char* msg7 = "not support distinct mixed with join"; - const char* msg8 = "not support distinct mixed with groupby"; - const char* msg9 = "not support distinct in nest query"; + const char* msg6 = "not support distinct mixed with join"; + const char* msg7 = "not support distinct mixed with groupby"; + const char* msg8 = "not support distinct in nest query"; // too many result columns not support order by in query if (taosArrayGetSize(pSelNodeList) > TSDB_MAX_COLUMNS) { @@ -2047,20 +2046,21 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS bool hasDistinct = false; bool hasAgg = false; - size_t numOfExpr = taosArrayGetSize(pSelNodeList); + size_t numOfExpr = taosArrayGetSize(pSelNodeList); int32_t distIdx = -1; for (int32_t i = 0; i < numOfExpr; ++i) { int32_t outputIndex = (int32_t)tscNumOfExprs(pQueryInfo); tSqlExprItem* pItem = taosArrayGet(pSelNodeList, i); - if (hasDistinct == false) { - hasDistinct = (pItem->distinct == true); - distIdx = i; - } + hasDistinct = (pItem->distinct == true); + distIdx = hasDistinct ? i : -1; + } int32_t type = pItem->pNode->type; if (type == SQL_NODE_SQLFUNCTION) { - hasAgg = true; + hasAgg = true; + if (hasDistinct) break; + pItem->pNode->functionId = isValidFunction(pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n); SUdfInfo* pUdfInfo = NULL; if (pItem->pNode->functionId < 0) { @@ -2100,20 +2100,17 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS //TODO(dengyihao), refactor as function //handle distinct func mixed with other func if (hasDistinct == true) { - if (distIdx != 0) { + if (distIdx != 0 || hasAgg) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } - if (hasAgg) { + if (joinQuery) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } - if (joinQuery) { + if (pQueryInfo->groupbyExpr.numOfGroupCols != 0) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } - if (pQueryInfo->groupbyExpr.numOfGroupCols != 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); - } if (pQueryInfo->pDownstream != NULL) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); } pQueryInfo->distinct = true; } From 90e0005c57f88d2f750d3594eae319eec3aa0b96 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 13 Aug 2021 11:16:57 +0800 Subject: [PATCH 54/75] [td-255] sleep a little bit longer before checking the status of dnodes in the clusters. --- tests/script/unique/dnode/alternativeRole.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/unique/dnode/alternativeRole.sim b/tests/script/unique/dnode/alternativeRole.sim index 955b757f06..14a6e92f06 100644 --- a/tests/script/unique/dnode/alternativeRole.sim +++ b/tests/script/unique/dnode/alternativeRole.sim @@ -30,7 +30,7 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 5000 sql show dnodes print dnode1 $data5_1 From 93eecb6c4b83041585dde6b1ee249a4f5c91cf10 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Fri, 13 Aug 2021 14:37:10 +0800 Subject: [PATCH 55/75] [TD-5722] removing bracket as it may cause error in the future --- tests/pytest/query/distinctOneColTb.py | 63 ++++++++++---------------- 1 file changed, 24 insertions(+), 39 deletions(-) diff --git a/tests/pytest/query/distinctOneColTb.py b/tests/pytest/query/distinctOneColTb.py index 1f6089629a..1570d3b4a5 100644 --- a/tests/pytest/query/distinctOneColTb.py +++ b/tests/pytest/query/distinctOneColTb.py @@ -25,116 +25,101 @@ class TDTestCase: def tb_all_query(self, num, sql="tb_all", where=""): tdSql.query( - f"select distinct(ts2) from {sql} {where}") + f"select distinct ts2 from {sql} {where}") tdSql.checkRows(num) tdSql.query( - f"select distinct(cint) from {sql} {where}") + f"select distinct cint from {sql} {where}") tdSql.checkRows(num) tdSql.query( - f"select distinct(cbigint) from {sql} {where}") + f"select distinct cbigint from {sql} {where}") tdSql.checkRows(num) tdSql.query( - f"select distinct(csmallint) from {sql} {where}") + f"select distinct csmallint from {sql} {where}") tdSql.checkRows(num) tdSql.query( - f"select distinct(ctinyint) from {sql} {where}") + f"select distinct ctinyint from {sql} {where}") tdSql.checkRows(num) tdSql.query( - f"select distinct(cfloat) from {sql} {where}") + f"select distinct cfloat from {sql} {where}") tdSql.checkRows(num) tdSql.query( - f"select distinct(cdouble) from {sql} {where}") + f"select distinct cdouble from {sql} {where}") tdSql.checkRows(num) tdSql.query( - f"select distinct(cbool) from {sql} {where}") + f"select distinct cbool from {sql} {where}") if num < 2: tdSql.checkRows(num) else: tdSql.checkRows(2) tdSql.query( - f"select distinct(cbinary) from {sql} {where}") + f"select distinct cbinary from {sql} {where}") tdSql.checkRows(num) tdSql.query( - f"select distinct(cnchar) from {sql} {where}") + f"select distinct cnchar from {sql} {where}") tdSql.checkRows(num) tdSql.query( - f"select distinct(ts2) as a from {sql} {where}") + f"select distinct ts2 as a from {sql} {where}") tdSql.checkRows(num) tdSql.query( - f"select distinct(cint) as a from {sql} {where}") + f"select distinct cint as a from {sql} {where}") tdSql.checkRows(num) tdSql.query( - f"select distinct(cbigint) as a from {sql} {where}") + f"select distinct cbigint as a from {sql} {where}") tdSql.checkRows(num) tdSql.query( - f"select distinct(csmallint) as a from {sql} {where}") + f"select distinct csmallint as a from {sql} {where}") tdSql.checkRows(num) tdSql.query( - f"select distinct(ctinyint) as a from {sql} {where}") + f"select distinct ctinyint as a from {sql} {where}") tdSql.checkRows(num) tdSql.query( - f"select distinct(cfloat) as a from {sql} {where}") + f"select distinct cfloat as a from {sql} {where}") tdSql.checkRows(num) tdSql.query( - f"select distinct(cdouble) as a from {sql} {where}") + f"select distinct cdouble as a from {sql} {where}") tdSql.checkRows(num) tdSql.query( - f"select distinct(cbool) as a from {sql} {where}") + f"select distinct cbool as a from {sql} {where}") if num < 2: tdSql.checkRows(num) else: tdSql.checkRows(2) tdSql.query( - f"select distinct(cbinary) as a from {sql} {where}") + f"select distinct cbinary as a from {sql} {where}") tdSql.checkRows(num) tdSql.query( - f"select distinct(cnchar) as a from {sql} {where}") + f"select distinct cnchar as a from {sql} {where}") tdSql.checkRows(num) def tb_all_query_sub(self, num, sql="tb_all", where="",colName = "c1"): tdSql.query( - f"select distinct({colName}) from {sql} {where}") + f"select distinct {colName} from {sql} {where}") tdSql.checkRows(num) def errorCheck(self, sql='tb_all'): - tdSql.error(f"select distinct() from {sql}") tdSql.error(f"select distinct from {sql}") - tdSql.error(f"distinct ts2 from {sql}") - tdSql.error(f"distinct(ts2) from {sql}") + tdSql.error(f"distinct ts2 from {sql}") tdSql.error(f"distinct c1 from") - tdSql.error(f"distinct(c1) from") - tdSql.error(f"select distinct(ts2), avg(cint) from {sql}") tdSql.error(f"select distinct ts2, avg(cint) from {sql}") ## the following line is going to core dump - #tdSql.error(f"select avg(cint), distinct(ts2) from {sql}") #tdSql.error(f"select avg(cint), distinct ts2 from {sql}") - tdSql.error(f"select distinct(ts2), cint from {sql}") - tdSql.error(f"select distinct ts2, cint from {sql}") - tdSql.error(f"select distinct(ts2) from {sql} group by cint") tdSql.error(f"select distinct ts2 from {sql} group by cint") - tdSql.error(f"select distinct(ts2) from {sql} intervasl(1s)") tdSql.error(f"select distinct ts2 from {sql} interval(1s)") - tdSql.error(f"select distinct(ts2) from {sql} slimit 1 soffset 1") tdSql.error(f"select distinct ts2 from {sql} slimit 1 soffset 1") - tdSql.error(f"select distinct(ts2) from {sql} slimit 1") tdSql.error(f"select distinct ts2 from {sql} slimit 1") ##order by is not supported but not being prohibited - #tdSql.error(f"select distinct(ts2) from {sql} order by desc") #tdSql.error(f"select distinct ts2 from {sql} order by desc") - #tdSql.error(f"select distinct(ts2) from {sql} order by asc") #tdSql.error(f"select distinct ts2 from {sql} order by asc") ##distinct should not use on first ts, but it can be applied - #tdSql.error(f"select distinct(ts) from {sql}") #tdSql.error(f"select distinct ts from {sql}") ##distinct should not be used in inner query, but error did not occur - # tdSql.error(f"select distinct(ts2) from (select distinct ts2 from {sql})") # tdSql.error(f"select distinct ts2 from (select distinct ts2 from {sql})") @@ -208,7 +193,7 @@ class TDTestCase: tdSql.prepare() tdLog.notice( - "==============phase1 distinct(col1) with no values==========") + "==============phase1 distinct col1 with no values==========") tdSql.execute("create stable if not exists stb_all (ts timestamp, ts2 timestamp, cint int, cbigint bigint, csmallint smallint, ctinyint tinyint,cfloat float, cdouble double, cbool bool, cbinary binary(32), cnchar nchar(32)) tags(tint int)") tdSql.execute( "create table if not exists tb_all using stb_all tags(1)") @@ -218,7 +203,7 @@ class TDTestCase: "==============phase1 finished ==========\n\n\n") tdLog.notice( - "==============phase2 distinct(col1) all values are null==========") + "==============phase2 distinct col1 all values are null==========") tdSql.execute( "insert into tb_all values(now,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)") tdSql.execute( From 96dd315a26fc6d77050653347b7dd795bc06aa12 Mon Sep 17 00:00:00 2001 From: XYWang Date: Fri, 13 Aug 2021 18:06:25 +0800 Subject: [PATCH 56/75] [TD-6039]: fixed test case error --- tests/pytest/client/client.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/pytest/client/client.py b/tests/pytest/client/client.py index c559860d3c..0fc6741131 100644 --- a/tests/pytest/client/client.py +++ b/tests/pytest/client/client.py @@ -35,6 +35,8 @@ class TDTestCase: ret = tdSql.query('select server_status() as result') tdSql.checkData(0, 0, 1) + time.sleep(1) + ret = tdSql.query('show dnodes') dnodeId = tdSql.getData(0, 0); @@ -43,6 +45,7 @@ class TDTestCase: ret = tdSql.execute('alter dnode "%s" debugFlag 135' % dnodeId) tdLog.info('alter dnode "%s" debugFlag 135 -> ret: %d' % (dnodeId, ret)) + time.sleep(1) ret = tdSql.query('show mnodes') tdSql.checkRows(1) @@ -63,6 +66,9 @@ class TDTestCase: tdSql.execute('create stable st (ts timestamp, f int) tags(t int)') tdSql.execute('create table ct1 using st tags(1)'); tdSql.execute('create table ct2 using st tags(2)'); + + time.sleep(1) + ret = tdSql.query('show vnodes "{}"'.format(dnodeEndpoint)) tdSql.checkRows(1) tdSql.checkData(0, 0, 2) From 52c5aa04e299155aab23d63deff90cead01301f2 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 13 Aug 2021 18:17:36 +0800 Subject: [PATCH 57/75] update --- tests/pytest/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 137069e6b6..377913423a 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -284,7 +284,7 @@ python3 ./test.py -f alter/alterTabAddTagWithNULL.py python3 ./test.py -f alter/alterTimestampColDataProcess.py # client -#python3 ./test.py -f client/client.py +python3 ./test.py -f client/client.py python3 ./test.py -f client/version.py python3 ./test.py -f client/alterDatabase.py python3 ./test.py -f client/noConnectionErrorTest.py From 823e0d2614c9e8f7de05c0c0f19cff1d6c954eec Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 13 Aug 2021 22:46:47 +0800 Subject: [PATCH 58/75] [TD-6044]: WAL compatibility since v2.1.5.0 --- src/inc/twal.h | 2 +- src/wal/src/walWrite.c | 99 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 94 insertions(+), 7 deletions(-) diff --git a/src/inc/twal.h b/src/inc/twal.h index bce398d6f9..868a1fbd78 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -32,7 +32,7 @@ typedef enum { typedef struct { int8_t msgType; - int8_t sver; + int8_t sver; // sver 2 for WAL SDataRow/SMemRow compatibility int8_t reserved[2]; int32_t len; uint64_t version; diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 4774998799..2dfdb84818 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -17,6 +17,7 @@ #define TAOS_RANDOM_FILE_FAIL_TEST #include "os.h" #include "taoserror.h" +#include "taosmsg.h" #include "tchecksum.h" #include "tfile.h" #include "twal.h" @@ -114,7 +115,7 @@ void walRemoveAllOldFiles(void *handle) { #if defined(WAL_CHECKSUM_WHOLE) static void walUpdateChecksum(SWalHead *pHead) { - pHead->sver = 1; + pHead->sver = 2; pHead->cksum = 0; pHead->cksum = taosCalcChecksum(0, (uint8_t *)pHead, sizeof(*pHead) + pHead->len); } @@ -122,7 +123,7 @@ static void walUpdateChecksum(SWalHead *pHead) { static int walValidateChecksum(SWalHead *pHead) { if (pHead->sver == 0) { // for compatible with wal before sver 1 return taosCheckChecksumWhole((uint8_t *)pHead, sizeof(*pHead)); - } else if (pHead->sver == 1) { + } else if (pHead->sver >= 1) { uint32_t cksum = pHead->cksum; pHead->cksum = 0; return taosCheckChecksum((uint8_t *)pHead, sizeof(*pHead) + pHead->len, cksum); @@ -281,7 +282,7 @@ static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int64_t tfd, return TSDB_CODE_SUCCESS; } - if (pHead->sver == 1) { + if (pHead->sver >= 1) { if (tfRead(tfd, pHead->cont, pHead->len) < pHead->len) { wError("vgId:%d, read to end of corrupted wal file, offset:%" PRId64, pWal->vgId, pos); return TSDB_CODE_WAL_FILE_CORRUPTED; @@ -306,7 +307,88 @@ static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int64_t tfd, return TSDB_CODE_WAL_FILE_CORRUPTED; } +// Add SMemRowType ahead of SDataRow +static void expandSubmitBlk(SSubmitBlk *pDest, SSubmitBlk *pSrc, int32_t *lenExpand) { + memcpy(pDest, pSrc, sizeof(SSubmitBlk)); + int nRows = htons(pSrc->numOfRows); + if (nRows <= 0) { + return; + } + char *pDestData = pDest->data; + char *pSrcData = pSrc->data; + for (int i = 0; i < nRows; ++i) { + memRowSetType(pDestData, SMEM_ROW_DATA); + memcpy(memRowDataBody(pDestData), pSrcData, dataRowLen(pSrcData)); + pDestData = POINTER_SHIFT(pDestData, memRowTLen(pDestData)); + pSrcData = POINTER_SHIFT(pSrcData, dataRowLen(pSrcData)); + ++(*lenExpand); + } + int32_t dataLen = htonl(pDest->dataLen); + pDest->dataLen = htonl(dataLen + nRows * sizeof(uint8_t)); +} +static bool walIsSDataRow(void *pBlkData, int nRows, int32_t dataLen) { + int32_t len = 0; + for (int i = 0; i < nRows; ++i) { + len += dataRowLen(pBlkData); + if (len > dataLen) { + return false; + } + pBlkData = POINTER_SHIFT(pBlkData, dataRowLen(pBlkData)); + } + if (len != dataLen) { + return false; + } + return true; +} +// for WAL SMemRow/SDataRow compatibility +static int walSMemRowCheck(SWalHead *pHead) { + if ((pHead->sver < 2) && (pHead->msgType == TSDB_MSG_TYPE_SUBMIT)) { + SSubmitMsg *pMsg = (SSubmitMsg *)pHead->cont; + int32_t numOfBlocks = htonl(pMsg->numOfBlocks); + if (numOfBlocks <= 0) { + return 0; + } + + int32_t nTotalRows = 0; + SSubmitBlk *pBlk = (SSubmitBlk *)pMsg->blocks; + for (int32_t i = 0; i < numOfBlocks; ++i) { + int32_t dataLen = htonl(pBlk->dataLen); + int32_t nRows = htons(pBlk->numOfRows); + nTotalRows += nRows; + if (!walIsSDataRow(pBlk->data, nRows, dataLen)) { + return 0; + } + pBlk = (SSubmitBlk *)POINTER_SHIFT(pBlk, sizeof(SSubmitBlk) + dataLen); + } + + SWalHead *pWalHead = (SWalHead *)calloc(sizeof(SWalHead) + pHead->len + nTotalRows * sizeof(uint8_t), 1); + if (pWalHead == NULL) { + return -1; + } + + memcpy(pWalHead, pHead, sizeof(SWalHead) + sizeof(SSubmitMsg)); + + SSubmitMsg *pDestMsg = (SSubmitMsg *)pWalHead->cont; + SSubmitBlk *pDestBlks = (SSubmitBlk *)pDestMsg->blocks; + SSubmitBlk *pSrcBlks = (SSubmitBlk *)pMsg->blocks; + int32_t lenExpand = 0; + for (int32_t i = 0; i < numOfBlocks; ++i) { + expandSubmitBlk(pDestBlks, pSrcBlks, &lenExpand); + pDestBlks = POINTER_SHIFT(pDestBlks, htonl(pDestBlks->dataLen) + sizeof(SSubmitBlk)); + pSrcBlks = POINTER_SHIFT(pSrcBlks, htonl(pSrcBlks->dataLen) + sizeof(SSubmitBlk)); + } + if (lenExpand > 0) { + pDestMsg->header.contLen = htonl(pDestMsg->length) + lenExpand; + pDestMsg->length = htonl(pDestMsg->header.contLen); + pWalHead->len = pWalHead->len + lenExpand; + } + + memcpy(pHead, pWalHead, sizeof(SWalHead) + pWalHead->len); + tfree(pWalHead); + } + return 0; +} static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId) { int32_t size = WAL_MAX_SIZE; @@ -346,7 +428,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch } #if defined(WAL_CHECKSUM_WHOLE) - if ((pHead->sver == 0 && !walValidateChecksum(pHead)) || pHead->sver < 0 || pHead->sver > 1) { + if ((pHead->sver == 0 && !walValidateChecksum(pHead)) || pHead->sver < 0 || pHead->sver > 2) { wError("vgId:%d, file:%s, wal head cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, pHead->version, pHead->len, offset); code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset); @@ -379,7 +461,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch continue; } - if (pHead->sver == 1 && !walValidateChecksum(pHead)) { + if ((pHead->sver >= 1) && !walValidateChecksum(pHead)) { wError("vgId:%d, file:%s, wal whole cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, pHead->version, pHead->len, offset); code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset); @@ -431,7 +513,12 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch pWal->version = pHead->version; - //wInfo("writeFp: %ld", offset); + // wInfo("writeFp: %ld", offset); + if (0 != walSMemRowCheck(pHead)) { + wError("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d offset:%" PRId64, + pWal->vgId, fileId, pHead->version, pWal->version, pHead->len, offset); + return TAOS_SYSTEM_ERROR(errno); + } (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL, NULL); } From 11b0755ef56006fd4256316ed2f1b7e7b4817b93 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 14 Aug 2021 08:48:49 +0800 Subject: [PATCH 59/75] do further check for blk with len 0 and SKVRow --- src/wal/src/walWrite.c | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 2dfdb84818..7523369dc2 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -309,36 +309,63 @@ static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int64_t tfd, } // Add SMemRowType ahead of SDataRow static void expandSubmitBlk(SSubmitBlk *pDest, SSubmitBlk *pSrc, int32_t *lenExpand) { + // copy the header firstly memcpy(pDest, pSrc, sizeof(SSubmitBlk)); - int nRows = htons(pSrc->numOfRows); - if (nRows <= 0) { + + int32_t nRows = htons(pDest->numOfRows); + int32_t dataLen = htonl(pDest->dataLen); + + if ((nRows <= 0) || (dataLen <= 0)) { return; } + char *pDestData = pDest->data; char *pSrcData = pSrc->data; - for (int i = 0; i < nRows; ++i) { + for (int32_t i = 0; i < nRows; ++i) { memRowSetType(pDestData, SMEM_ROW_DATA); memcpy(memRowDataBody(pDestData), pSrcData, dataRowLen(pSrcData)); pDestData = POINTER_SHIFT(pDestData, memRowTLen(pDestData)); pSrcData = POINTER_SHIFT(pSrcData, dataRowLen(pSrcData)); ++(*lenExpand); } - int32_t dataLen = htonl(pDest->dataLen); pDest->dataLen = htonl(dataLen + nRows * sizeof(uint8_t)); } +// Check SDataRow by comparing the SDataRow len and SSubmitBlk dataLen static bool walIsSDataRow(void *pBlkData, int nRows, int32_t dataLen) { - int32_t len = 0; + if ((nRows <= 0) || (dataLen <= 0)) { + return true; + } + int32_t len = 0, kvLen = 0; for (int i = 0; i < nRows; ++i) { len += dataRowLen(pBlkData); if (len > dataLen) { return false; } + + /** + * For SDataRow between version [2.1.5.0 and 2.1.6.X], it would never conflict. + * For SKVRow between version [2.1.5.0 and 2.1.6.X], it may conflict in below scenario + * - with 1st type byte 0x01 and sversion 0x0101(257), thus do further check + */ + if (dataRowLen(pBlkData) == 257) { + SMemRow memRow = pBlkData; + SKVRow kvRow = memRowKvBody(memRow); + int nCols = kvRowNCols(kvRow); + uint16_t calcTsOffset = (uint16_t)(TD_MEM_ROW_KV_HEAD_SIZE + sizeof(SColIdx) * nCols); + uint16_t realTsOffset = (kvRowColIdx(kvRow))->offset; + if (calcTsOffset == realTsOffset) { + kvLen += memRowKvTLen(memRow); + } + } pBlkData = POINTER_SHIFT(pBlkData, dataRowLen(pBlkData)); } if (len != dataLen) { return false; } + if (kvLen == dataLen) { + return false; + } return true; } // for WAL SMemRow/SDataRow compatibility From 868ad35059d9012333adb20d99075ea59e6be97f Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Sat, 14 Aug 2021 09:41:30 +0800 Subject: [PATCH 60/75] add join.sim --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 5e2488dddd..6ad6a74eed 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -80,7 +80,7 @@ cd ../../../debug; make ./test.sh -f general/parser/set_tag_vals.sim ./test.sh -f general/parser/tags_filter.sim ./test.sh -f general/parser/slimit_alter_tags.sim -#./test.sh -f general/parser/join.sim +./test.sh -f general/parser/join.sim ./test.sh -f general/parser/join_multivnode.sim ./test.sh -f general/parser/binary_escapeCharacter.sim ./test.sh -f general/parser/repeatAlter.sim From 4fcb2b07eff360a368db4f469c37e3972696b06a Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 14 Aug 2021 09:42:35 +0800 Subject: [PATCH 61/75] update header --- src/wal/src/walWrite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 7523369dc2..9590aba224 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -352,7 +352,7 @@ static bool walIsSDataRow(void *pBlkData, int nRows, int32_t dataLen) { SMemRow memRow = pBlkData; SKVRow kvRow = memRowKvBody(memRow); int nCols = kvRowNCols(kvRow); - uint16_t calcTsOffset = (uint16_t)(TD_MEM_ROW_KV_HEAD_SIZE + sizeof(SColIdx) * nCols); + uint16_t calcTsOffset = (uint16_t)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nCols); uint16_t realTsOffset = (kvRowColIdx(kvRow))->offset; if (calcTsOffset == realTsOffset) { kvLen += memRowKvTLen(memRow); From 62987c39d277e24a0ca09b87b75da97beefb7234 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 14 Aug 2021 10:16:35 +0800 Subject: [PATCH 62/75] Hotfix/sangshuduo/td 5702 taosdemo remove memop (#7358) * [TD-5702]: taosdemo remove memory operation. * add remainderBufLen to check row data generation. * row data generation with remainder buffer length checking. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 38 +++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 3b91de32b0..f3270a22f2 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1340,8 +1340,8 @@ static char *rand_bool_str(){ static int32_t rand_bool(){ static int cursor; cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return g_randint[cursor] % 2; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randint[cursor % MAX_PREPARED_RAND] % 2; } static char *rand_tinyint_str() @@ -1356,8 +1356,8 @@ static int32_t rand_tinyint() { static int cursor; cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return g_randint[cursor] % 128; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randint[cursor % MAX_PREPARED_RAND] % 128; } static char *rand_smallint_str() @@ -1372,8 +1372,8 @@ static int32_t rand_smallint() { static int cursor; cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return g_randint[cursor] % 32767; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randint[cursor % MAX_PREPARED_RAND] % 32767; } static char *rand_int_str() @@ -1388,8 +1388,8 @@ static int32_t rand_int() { static int cursor; cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return g_randint[cursor]; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randint[cursor % MAX_PREPARED_RAND]; } static char *rand_bigint_str() @@ -1404,8 +1404,8 @@ static int64_t rand_bigint() { static int cursor; cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return g_randbigint[cursor]; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randbigint[cursor % MAX_PREPARED_RAND]; } static char *rand_float_str() @@ -1421,8 +1421,8 @@ static float rand_float() { static int cursor; cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return g_randfloat[cursor]; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randfloat[cursor % MAX_PREPARED_RAND]; } static char *demo_current_float_str() @@ -1437,8 +1437,9 @@ static float UNUSED_FUNC demo_current_float() { static int cursor; cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return (float)(9.8 + 0.04 * (g_randint[cursor] % 10) + g_randfloat[cursor]/1000000000); + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return (float)(9.8 + 0.04 * (g_randint[cursor % MAX_PREPARED_RAND] % 10) + + g_randfloat[cursor % MAX_PREPARED_RAND]/1000000000); } static char *demo_voltage_int_str() @@ -1453,8 +1454,8 @@ static int32_t UNUSED_FUNC demo_voltage_int() { static int cursor; cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return 215 + g_randint[cursor] % 10; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return 215 + g_randint[cursor % MAX_PREPARED_RAND] % 10; } static char *demo_phase_float_str() { @@ -1467,8 +1468,9 @@ static char *demo_phase_float_str() { static float UNUSED_FUNC demo_phase_float(){ static int cursor; cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return (float)((115 + g_randint[cursor] % 10 + g_randfloat[cursor]/1000000000)/360); + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return (float)((115 + g_randint[cursor % MAX_PREPARED_RAND] % 10 + + g_randfloat[cursor % MAX_PREPARED_RAND]/1000000000)/360); } #if 0 From b5c505a6b821092b6062a0e60b4fcb223d6b8b02 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 14 Aug 2021 11:11:20 +0800 Subject: [PATCH 63/75] code optimization --- src/wal/src/walWrite.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 9590aba224..e991bf02aa 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -388,7 +388,7 @@ static int walSMemRowCheck(SWalHead *pHead) { } pBlk = (SSubmitBlk *)POINTER_SHIFT(pBlk, sizeof(SSubmitBlk) + dataLen); } - + ASSERT(nTotalRows >= 0); SWalHead *pWalHead = (SWalHead *)calloc(sizeof(SWalHead) + pHead->len + nTotalRows * sizeof(uint8_t), 1); if (pWalHead == NULL) { return -1; @@ -544,6 +544,8 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch if (0 != walSMemRowCheck(pHead)) { wError("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, fileId, pHead->version, pWal->version, pHead->len, offset); + tfClose(tfd); + tfree(buffer); return TAOS_SYSTEM_ERROR(errno); } (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL, NULL); From 003fb714887ee96a5f17ab4d96ee7e6dfa9d4e27 Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Sat, 14 Aug 2021 12:03:23 +0800 Subject: [PATCH 64/75] [TD-6078]: fix binary/nchar null error in node.js [ci skip] (#7365) --- src/connector/nodejs/nodetaos/cinterface.js | 26 ++++++++++++++-- src/connector/nodejs/test/testnchar.js | 33 +++++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 src/connector/nodejs/test/testnchar.js diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/src/connector/nodejs/nodetaos/cinterface.js index 03d27e5593..5ba2739c35 100644 --- a/src/connector/nodejs/nodetaos/cinterface.js +++ b/src/connector/nodejs/nodetaos/cinterface.js @@ -109,6 +109,24 @@ function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, precision = 0) return res; } +function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, precision = 0) { + data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); + let res = []; + + let currOffset = 0; + while (currOffset < data.length) { + let len = data.readIntLE(currOffset, 2); + let dataEntry = data.slice(currOffset + 2, currOffset + len + 2); //one entry in a row under a column; + if (dataEntry[0] == 255) { + res.push(null) + } else { + res.push(dataEntry.toString("utf-8")); + } + currOffset += nbytes; + } + return res; +} + function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, precision = 0) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; @@ -117,7 +135,11 @@ function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, precision = 0) while (currOffset < data.length) { let len = data.readIntLE(currOffset, 2); let dataEntry = data.slice(currOffset + 2, currOffset + len + 2); //one entry in a row under a column; - res.push(dataEntry.toString("utf-8")); + if (dataEntry[0] == 255 && dataEntry[1] == 255) { + res.push(null) + } else { + res.push(dataEntry.toString("utf-8")); + } currOffset += nbytes; } return res; @@ -132,7 +154,7 @@ let convertFunctions = { [FieldTypes.C_BIGINT]: convertBigint, [FieldTypes.C_FLOAT]: convertFloat, [FieldTypes.C_DOUBLE]: convertDouble, - [FieldTypes.C_BINARY]: convertNchar, + [FieldTypes.C_BINARY]: convertBinary, [FieldTypes.C_TIMESTAMP]: convertTimestamp, [FieldTypes.C_NCHAR]: convertNchar } diff --git a/src/connector/nodejs/test/testnchar.js b/src/connector/nodejs/test/testnchar.js new file mode 100644 index 0000000000..68fad89c22 --- /dev/null +++ b/src/connector/nodejs/test/testnchar.js @@ -0,0 +1,33 @@ +const taos = require('../tdengine'); +var conn = taos.connect({ host: "localhost" }); +var c1 = conn.cursor(); + + +function checkData(data, row, col, expect) { + let checkdata = data[row][col]; + if (checkdata == expect) { + // console.log('check pass') + } + else { + console.log('check failed, expect ' + expect + ', but is ' + checkdata) + } +} + +c1.execute('drop database if exists testnodejsnchar') +c1.execute('create database testnodejsnchar') +c1.execute('use testnodejsnchar'); +c1.execute('create table tb (ts timestamp, value float, text binary(200))') +c1.execute("insert into tb values('2021-06-10 00:00:00', 24.7, '中文10000000000000000000000');") - +c1.execute('insert into tb values(1623254400150, 24.7, NULL);') +c1.execute('import into tb values(1623254400300, 24.7, "中文3中文10000000000000000000000中文10000000000000000000000中文10000000000000000000000中文10000000000000000000000");') +sql = 'select * from tb;' + +console.log('*******************************************') + +c1.execute(sql); +data = c1.fetchall(); +console.log(data) +//check data about insert data +checkData(data, 0, 2, '中文10000000000000000000000') +checkData(data, 1, 2, null) +checkData(data, 2, 2, '中文3中文10000000000000000000000中文10000000000000000000000中文10000000000000000000000中文10000000000000000000000') \ No newline at end of file From 87396aaba12133032cf5336a2dd7930cbbd5de5b Mon Sep 17 00:00:00 2001 From: Keli Date: Sat, 14 Aug 2021 12:06:26 +0800 Subject: [PATCH 65/75] fix README.MD in minidevops directory (#7364) Signed-off-by: Keli --- minidevops/README.MD | 4 ---- 1 file changed, 4 deletions(-) diff --git a/minidevops/README.MD b/minidevops/README.MD index 9937ad04ad..dcc8c2fa96 100644 --- a/minidevops/README.MD +++ b/minidevops/README.MD @@ -218,8 +218,4 @@ use telegraf； 使用telegraf这个数据库。然后执行show tables，describe table等命令详细查询下telegraf这个库里保存了些什么数据。 具体TDengine的查询语句可以参考[TDengine官方文档](https://www.taosdata.com/cn/documentation/taos-sql/) ## 接入多个监控对象 -<<<<<<< HEAD 就像前面原理介绍的，这个miniDevops的小系统，已经提供了一个时序数据库和可视化系统，对于多台机器的监控，只需要将每台机器的telegraf或prometheus配置按上面所述修改，就可以完成监控数据采集和可视化呈现了。 -======= -就像前面原理介绍的，这个miniDevops的小系统，已经提供了一个时序数据库和可视化系统，对于多台机器的监控，只需要将每台机器的telegraf或prometheus配置按上面所述修改，就可以完成监控数据采集和可视化呈现了。 ->>>>>>> 740f82af58c4ecc2deecfa36fb1de4ef5ee55efc From cc4db8915253469d22d3def68e628ec71be38a30 Mon Sep 17 00:00:00 2001 From: wpan Date: Sat, 14 Aug 2021 16:26:39 +0800 Subject: [PATCH 66/75] improve performance --- src/client/src/tscUtil.c | 13 +- src/query/inc/qExecutor.h | 1 - src/query/inc/qFilter.h | 14 +- src/query/src/qExecutor.c | 27 +- src/query/src/qFilter.c | 374 ++++++++++++++---- tests/script/general/parser/condition.sim | 42 ++ .../script/general/parser/condition_query.sim | 111 ++++++ 7 files changed, 469 insertions(+), 113 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 82f6827909..991b1ad50e 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -821,17 +821,22 @@ static void doSetupSDataBlock(SSqlRes* pRes, SSDataBlock* pBlock, SFilterInfo* p // filter data if needed if (pFilterInfo) { //doSetFilterColumnInfo(pFilterInfo, numOfFilterCols, pBlock); - doSetFilterColInfo(pFilterInfo, pBlock); + filterSetColFieldData(pFilterInfo, pBlock->info.numOfCols, pBlock->pDataBlock); bool gotNchar = false; filterConverNcharColumns(pFilterInfo, pBlock->info.rows, &gotNchar); - int8_t* p = calloc(pBlock->info.rows, sizeof(int8_t)); + int8_t* p = NULL; //bool all = doFilterDataBlock(pFilterInfo, numOfFilterCols, pBlock->info.rows, p); - bool all = filterExecute(pFilterInfo, pBlock->info.rows, p); + bool all = filterExecute(pFilterInfo, pBlock->info.rows, &p, NULL, 0); if (gotNchar) { filterFreeNcharColumns(pFilterInfo); } if (!all) { - doCompactSDataBlock(pBlock, pBlock->info.rows, p); + if (p) { + doCompactSDataBlock(pBlock, pBlock->info.rows, p); + } else { + pBlock->info.rows = 0; + pBlock->pBlockStatis = NULL; // clean the block statistics info + } } tfree(p); diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 996d925756..449cf4119f 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -578,7 +578,6 @@ SSDataBlock* doSLimit(void* param, bool* newgroup); int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock); -void doSetFilterColInfo(SFilterInfo *pFilters, SSDataBlock* pBlock); bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p); void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); diff --git a/src/query/inc/qFilter.h b/src/query/inc/qFilter.h index 38da341c59..af45b816f9 100644 --- a/src/query/inc/qFilter.h +++ b/src/query/inc/qFilter.h @@ -31,11 +31,10 @@ extern "C" { #define FILTER_DEFAULT_GROUP_UNIT_SIZE 2 #define FILTER_DUMMY_EMPTY_OPTR 127 -#define FILTER_DUMMY_RANGE_OPTR 126 #define MAX_NUM_STR_SIZE 40 -#define FILTER_RM_UNIT_MIN_ROWS 1024 +#define FILTER_RM_UNIT_MIN_ROWS 100 enum { FLD_TYPE_COLUMN = 1, @@ -106,7 +105,7 @@ typedef struct SFilterColRange { typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t); typedef int32_t(*filter_desc_compare_func)(const void *, const void *); -typedef bool(*filter_exec_func)(void *, int32_t, int8_t*); +typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SDataStatis *, int16_t); typedef struct SFilterRangeCompare { int64_t s; @@ -278,12 +277,13 @@ typedef struct SFilterInfo { #define CHK_RET(c, r) do { if (c) { return r; } } while (0) #define CHK_JMP(c) do { if (c) { goto _return; } } while (0) #define CHK_LRETV(c,...) do { if (c) { qError(__VA_ARGS__); return; } } while (0) -#define CHK_LRET(c, r,...) do { if (c) { qError(__VA_ARGS__); return r; } } while (0) +#define CHK_LRET(c, r,...) do { if (c) { if (r) {qError(__VA_ARGS__); } else { qDebug(__VA_ARGS__); } return r; } } while (0) #define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx])) #define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx])) #define FILTER_GET_COL_FIELD_TYPE(fi) (((SSchema *)((fi)->desc))->type) #define FILTER_GET_COL_FIELD_SIZE(fi) (((SSchema *)((fi)->desc))->bytes) +#define FILTER_GET_COL_FIELD_ID(fi) (((SSchema *)((fi)->desc))->colId) #define FILTER_GET_COL_FIELD_DESC(fi) ((SSchema *)((fi)->desc)) #define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri)) #define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType) @@ -293,10 +293,12 @@ typedef struct SFilterInfo { #define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid]) #define FILTER_UNIT_LEFT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->left) #define FILTER_UNIT_RIGHT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right) +#define FILTER_UNIT_RIGHT2_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right2) #define FILTER_UNIT_DATA_TYPE(u) ((u)->compare.type) #define FILTER_UNIT_COL_DESC(i, u) FILTER_GET_COL_FIELD_DESC(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_COL_DATA(i, u, ri) FILTER_GET_COL_FIELD_DATA(FILTER_UNIT_LEFT_FIELD(i, u), ri) #define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u)) +#define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) #define FILTER_UNIT_COL_IDX(u) ((u)->left.idx) #define FILTER_UNIT_OPTR(u) ((u)->compare.optr) @@ -322,8 +324,8 @@ typedef struct SFilterInfo { extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options); -extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t* p); -extern int32_t filterSetColFieldData(SFilterInfo *info, int16_t colId, void *data); +extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols); +extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock); extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 22e4f87ef9..67f64ba0aa 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2774,12 +2774,13 @@ void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInf void filterColRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) { int32_t numOfRows = pBlock->info.rows; - int8_t *p = calloc(numOfRows, sizeof(int8_t)); + int8_t *p = NULL; bool all = true; if (pRuntimeEnv->pTsBuf != NULL) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); + p = calloc(numOfRows, sizeof(int8_t)); + TSKEY* k = (TSKEY*) pColInfoData->pData; for (int32_t i = 0; i < numOfRows; ++i) { int32_t offset = ascQuery? i:(numOfRows - i - 1); @@ -2802,11 +2803,16 @@ void filterColRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock // save the cursor status pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); } else { - all = filterExecute(pRuntimeEnv->pQueryAttr->pFilters, numOfRows, p); + all = filterExecute(pRuntimeEnv->pQueryAttr->pFilters, numOfRows, &p, pBlock->pBlockStatis, pRuntimeEnv->pQueryAttr->numOfCols); } if (!all) { - doCompactSDataBlock(pBlock, numOfRows, p); + if (p) { + doCompactSDataBlock(pBlock, numOfRows, p); + } else { + pBlock->info.rows = 0; + pBlock->pBlockStatis = NULL; // clean the block statistics info + } } tfree(p); @@ -2855,15 +2861,6 @@ void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFi } } - -void doSetFilterColInfo(SFilterInfo * pFilters, SSDataBlock* pBlock) { - for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, j); - - filterSetColFieldData(pFilters, pColInfo->info.colId, pColInfo->pData); - } -} - int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { *status = BLK_DATA_NO_NEEDED; @@ -3003,7 +3000,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa } if (pQueryAttr->pFilters != NULL) { - doSetFilterColInfo(pQueryAttr->pFilters, pBlock); + filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock); } if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) { diff --git a/src/query/src/qFilter.c b/src/query/src/qFilter.c index f4aec03513..72f8376af6 100644 --- a/src/query/src/qFilter.c +++ b/src/query/src/qFilter.c @@ -1521,7 +1521,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) int32_t type = FILTER_UNIT_DATA_TYPE(unit); int32_t len = 0; int32_t tlen = 0; - char str[256] = {0}; + char str[512] = {0}; SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); SSchema *sch = left->desc; @@ -1539,6 +1539,24 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) strcat(str, "NULL"); } strcat(str, "]"); + + if (unit->compare.optr2) { + strcat(str, " && "); + sprintf(str + strlen(str), "[%d][%s] %s [", sch->colId, sch->name, gOptrStr[unit->compare.optr2].str); + + if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { + SFilterField *right = FILTER_UNIT_RIGHT2_FIELD(info, unit); + char *data = right->data; + if (IS_VAR_DATA_TYPE(type)) { + tlen = varDataLen(data); + data += VARSTR_HEADER_SIZE; + } + converToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen); + } else { + strcat(str, "NULL"); + } + strcat(str, "]"); + } qDebug("%s", str); //TODO } @@ -1556,37 +1574,63 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) return; } - qDebug("%s - RANGE info:", msg); + if (options == 1) { + qDebug("%s - RANGE info:", msg); - qDebug("RANGE Num:%u", info->colRangeNum); - for (uint16_t i = 0; i < info->colRangeNum; ++i) { - SFilterRangeCtx *ctx = info->colRange[i]; - qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange); - if (ctx->isrange) { - SFilterRangeNode *r = ctx->rs; - while (r) { - char str[256] = {0}; - int32_t tlen = 0; - if (FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { - strcat(str,"(NULL)"); - } else { - FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); - converToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen); - FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); + qDebug("RANGE Num:%u", info->colRangeNum); + for (uint16_t i = 0; i < info->colRangeNum; ++i) { + SFilterRangeCtx *ctx = info->colRange[i]; + qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange); + if (ctx->isrange) { + SFilterRangeNode *r = ctx->rs; + while (r) { + char str[256] = {0}; + int32_t tlen = 0; + if (FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { + strcat(str,"(NULL)"); + } else { + FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); + converToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen); + FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); + } + strcat(str, " - "); + if (FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { + strcat(str, "(NULL)"); + } else { + FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); + converToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen); + FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); + } + qDebug("range: %s", str); + + r = r->next; } - strcat(str, " - "); - if (FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { - strcat(str, "(NULL)"); - } else { - FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); - converToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen); - FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); - } - qDebug("range: %s", str); - - r = r->next; } } + + return; + } + + qDebug("%s - Block Filter info:", msg); + + if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) { + qDebug("Flag:%s", "ALL"); + return; + } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) { + qDebug("Flag:%s", "EMPTY"); + return; + } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE)){ + qDebug("Flag:%s", "ACTIVE"); + } + + qDebug("GroupNum:%d", info->blkGroupNum); + uint16_t *unitIdx = info->blkUnits; + for (uint16_t i = 0; i < info->blkGroupNum; ++i) { + qDebug("Group[%d] UnitNum: %d:", i, *unitIdx); + uint16_t unitNum = *(unitIdx++); + for (uint16_t m = 0; m < unitNum; ++m) { + qDebug("uidx[%d]", *(unitIdx++)); + } } } } @@ -1674,7 +1718,9 @@ void filterFreeInfo(SFilterInfo *info) { CHK_RETV(info == NULL); tfree(info->cunits); - + tfree(info->blkUnitRes); + tfree(info->blkUnits); + for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) { for (uint16_t f = 0; f < info->fields[i].num; ++f) { filterFreeField(&info->fields[i].fields[f], i); @@ -2485,7 +2531,9 @@ int32_t filterGenerateComInfo(SFilterInfo *info) { uint16_t n = 0; info->cunits = malloc(info->unitNum * sizeof(*info->cunits)); - + info->blkUnitRes = malloc(sizeof(*info->blkUnitRes) * info->unitNum); + info->blkUnits = malloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum); + for (uint16_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; @@ -2493,6 +2541,7 @@ int32_t filterGenerateComInfo(SFilterInfo *info) { info->cunits[i].rfunc = filterGetRangeCompFuncFromOptrs(unit->compare.optr, unit->compare.optr2); info->cunits[i].optr = FILTER_UNIT_OPTR(unit); info->cunits[i].colData = NULL; + info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit); if (unit->right.type == FLD_TYPE_VALUE) { info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); @@ -2540,6 +2589,7 @@ int32_t filterUpdateComUnits(SFilterInfo *info) { return TSDB_CODE_SUCCESS; } + int32_t filterRmUnitByRange(SFilterInfo *info, SDataStatis *pDataStatis, int32_t numOfCols, int32_t numOfRows) { int32_t rmUnit = 0; @@ -2590,7 +2640,9 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SDataStatis *pDataStatis, int32_t } } - if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL) { + if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL + || cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE + || cunit->optr == TSDB_RELATION_NOT_EQUAL) { continue; } @@ -2613,21 +2665,48 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SDataStatis *pDataStatis, int32_t if (cunit->rfunc >= 0) { minRes = (*gRangeCompare[cunit->rfunc])(minVal, minVal, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); maxRes = (*gRangeCompare[cunit->rfunc])(maxVal, maxVal, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); + + if (minRes && maxRes) { + info->blkUnitRes[k] = 1; + rmUnit = 1; + } else if ((!minRes) && (!maxRes)) { + minRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_LESS_EQUAL, minVal, cunit->valData); + maxRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_GREATER_EQUAL, maxVal, cunit->valData2); + + if (minRes && maxRes) { + continue; + } + + info->blkUnitRes[k] = -1; + rmUnit = 1; + } } else { minRes = filterDoCompare(gDataCompare[cunit->func], cunit->optr, minVal, cunit->valData); maxRes = filterDoCompare(gDataCompare[cunit->func], cunit->optr, maxVal, cunit->valData); + + if (minRes && maxRes) { + info->blkUnitRes[k] = 1; + rmUnit = 1; + } else if ((!minRes) && (!maxRes)) { + if (cunit->optr == TSDB_RELATION_EQUAL) { + minRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_GREATER, minVal, cunit->valData); + maxRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_LESS, maxVal, cunit->valData); + if (minRes || maxRes) { + info->blkUnitRes[k] = -1; + rmUnit = 1; + } + + continue; + } + + info->blkUnitRes[k] = -1; + rmUnit = 1; + } } - if (minRes && maxRes) { - info->blkUnitRes[k] = 1; - rmUnit = 1; - } else if ((!minRes) && (!maxRes)) { - info->blkUnitRes[k] = -1; - rmUnit = 1; - } } - CHK_RET(rmUnit == 0, TSDB_CODE_SUCCESS); + CHK_LRET(rmUnit == 0, TSDB_CODE_SUCCESS, "NO Block Filter APPLY"); info->blkGroupNum = info->groupNum; @@ -2643,11 +2722,11 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SDataStatis *pDataStatis, int32_t for (uint32_t u = 0; u < group->unitNum; ++u) { uint16_t uidx = group->unitIdxs[u]; - if (info->blkUnitRes[u] == 1) { + if (info->blkUnitRes[uidx] == 1) { --(*unitNum); all = 1; continue; - } else if (info->blkUnitRes[u] == -1) { + } else if (info->blkUnitRes[uidx] == -1) { *unitNum = 0; empty = 1; break; @@ -2681,43 +2760,147 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SDataStatis *pDataStatis, int32_t _return: - qDebug("Block Filter Result:%s", FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY) ? "EMPTY" : (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL) ? "ALL" : "FILTER")); + filterDumpInfoToString(info, "Block Filter", 2); return TSDB_CODE_SUCCESS; } - - -static FORCE_INLINE bool filterExecuteImplAll(void *info, int32_t numOfRows, int8_t* p) { - return true; -} -static FORCE_INLINE bool filterExecuteImplEmpty(void *info, int32_t numOfRows, int8_t* p) { - return false; -} -static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, int8_t* p) { +bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; - + uint16_t *unitIdx = NULL; + + *p = calloc(numOfRows, sizeof(int8_t)); + for (int32_t i = 0; i < numOfRows; ++i) { - uint16_t uidx = info->groups[0].unitIdxs[0]; - void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - p[i] = isNull(colData, info->cunits[uidx].dataType); - if (p[i] == 0) { + //FILTER_UNIT_CLR_F(info); + + unitIdx = info->blkUnits; + + for (uint32_t g = 0; g < info->blkGroupNum; ++g) { + uint16_t unitNum = *(unitIdx++); + for (uint32_t u = 0; u < unitNum; ++u) { + SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)]; + void *colData = (char *)cunit->colData + cunit->dataSize * i; + + //if (FILTER_UNIT_GET_F(info, uidx)) { + // p[i] = FILTER_UNIT_GET_R(info, uidx); + //} else { + uint8_t optr = cunit->optr; + + if (isNull(colData, cunit->dataType)) { + (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; + } else { + if (optr == TSDB_RELATION_NOTNULL) { + (*p)[i] = 1; + } else if (optr == TSDB_RELATION_ISNULL) { + (*p)[i] = 0; + } else if (cunit->rfunc >= 0) { + (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); + } else { + (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + } + + //FILTER_UNIT_SET_R(info, uidx, p[i]); + //FILTER_UNIT_SET_F(info, uidx); + } + + if ((*p)[i] == 0) { + break; + } + } + + if ((*p)[i]) { + break; + } + + unitIdx += unitNum; + } + + if ((*p)[i] == 0) { all = false; } } return all; } -static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows, int8_t* p) { + + + +int32_t filterExecuteBasedOnStatis(SFilterInfo *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols, bool* all) { + if (statis && numOfRows >= FILTER_RM_UNIT_MIN_ROWS) { + info->blkFlag = 0; + + filterRmUnitByRange(info, statis, numOfCols, numOfRows); + + if (info->blkFlag) { + if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) { + *all = true; + goto _return; + } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) { + *all = false; + goto _return; + } + + assert(info->unitNum > 1); + + *all = filterExecuteBasedOnStatisImpl(info, numOfRows, p, statis, numOfCols); + + goto _return; + } + } + + return 1; + +_return: + info->blkFlag = 0; + + return TSDB_CODE_SUCCESS; +} + + +static FORCE_INLINE bool filterExecuteImplAll(void *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { + return true; +} +static FORCE_INLINE bool filterExecuteImplEmpty(void *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { + return false; +} +static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; + + if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + return all; + } + + *p = calloc(numOfRows, sizeof(int8_t)); for (int32_t i = 0; i < numOfRows; ++i) { uint16_t uidx = info->groups[0].unitIdxs[0]; void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - p[i] = !isNull(colData, info->cunits[uidx].dataType); - if (p[i] == 0) { + (*p)[i] = isNull(colData, info->cunits[uidx].dataType); + if ((*p)[i] == 0) { + all = false; + } + } + + return all; +} +static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + + if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + return all; + } + + *p = calloc(numOfRows, sizeof(int8_t)); + + for (int32_t i = 0; i < numOfRows; ++i) { + uint16_t uidx = info->groups[0].unitIdxs[0]; + void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + (*p)[i] = !isNull(colData, info->cunits[uidx].dataType); + if ((*p)[i] == 0) { all = false; } } @@ -2725,7 +2908,7 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows return all; } -bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t* p) { +bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; uint16_t dataSize = info->cunits[0].dataSize; @@ -2734,6 +2917,12 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t* p) { void *valData = info->cunits[0].valData; void *valData2 = info->cunits[0].valData2; __compar_fn_t func = gDataCompare[info->cunits[0].func]; + + if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + return all; + } + + *p = calloc(numOfRows, sizeof(int8_t)); for (int32_t i = 0; i < numOfRows; ++i) { if (isNull(colData, info->cunits[0].dataType)) { @@ -2742,9 +2931,9 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t* p) { continue; } - p[i] = (*rfunc)(colData, colData, valData, valData2, func); + (*p)[i] = (*rfunc)(colData, colData, valData, valData2, func); - if (p[i] == 0) { + if ((*p)[i] == 0) { all = false; } @@ -2754,9 +2943,15 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t* p) { return all; } -bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t* p) { +bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; + + if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + return all; + } + + *p = calloc(numOfRows, sizeof(int8_t)); for (int32_t i = 0; i < numOfRows; ++i) { uint16_t uidx = info->groups[0].unitIdxs[0]; @@ -2766,9 +2961,9 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t* p) { continue; } - p[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); + (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); - if (p[i] == 0) { + if ((*p)[i] == 0) { all = false; } } @@ -2777,10 +2972,16 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t* p) { } -bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t* p) { +bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; + if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + return all; + } + + *p = calloc(numOfRows, sizeof(int8_t)); + for (int32_t i = 0; i < numOfRows; ++i) { //FILTER_UNIT_CLR_F(info); @@ -2797,33 +2998,33 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t* p) { uint8_t optr = cunit->optr; if (isNull(colData, cunit->dataType)) { - p[i] = optr == TSDB_RELATION_ISNULL ? true : false; + (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; } else { if (optr == TSDB_RELATION_NOTNULL) { - p[i] = 1; + (*p)[i] = 1; } else if (optr == TSDB_RELATION_ISNULL) { - p[i] = 0; + (*p)[i] = 0; } else if (cunit->rfunc >= 0) { - p[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); + (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); } else { - p[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); } //FILTER_UNIT_SET_R(info, uidx, p[i]); //FILTER_UNIT_SET_F(info, uidx); } - if (p[i] == 0) { + if ((*p)[i] == 0) { break; } } - if (p[i]) { + if ((*p)[i]) { break; } } - if (p[i] == 0) { + if ((*p)[i] == 0) { all = false; } } @@ -2831,8 +3032,9 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t* p) { return all; } -FORCE_INLINE bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t* p) { - return (*info->func)(info, numOfRows, p); + +FORCE_INLINE bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { + return (*info->func)(info, numOfRows, p, statis, numOfCols); } int32_t filterSetExecFunc(SFilterInfo *info) { @@ -2914,7 +3116,7 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t filterSetColFieldData(SFilterInfo *info, int16_t colId, void *data) { +int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock) { CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL"); CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds"); @@ -2925,10 +3127,14 @@ int32_t filterSetColFieldData(SFilterInfo *info, int16_t colId, void *data) { for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; SSchema* sch = fi->desc; - if (sch->colId == colId) { - fi->data = data; - - break; + + for (int32_t j = 0; j < numOfCols; ++j) { + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j); + if (sch->colId == pColInfo->info.colId) { + fi->data = pColInfo->pData; + + break; + } } } @@ -3078,12 +3284,6 @@ bool filterRangeExecute(SFilterInfo *info, SDataStatis *pDataStatis, int32_t num CHK_RET(!ret, ret); } - info->blkFlag = 0; - - if (ret && numOfRows >= FILTER_RM_UNIT_MIN_ROWS) { - filterRmUnitByRange(info, pDataStatis, numOfCols, numOfRows); - } - return ret; } diff --git a/tests/script/general/parser/condition.sim b/tests/script/general/parser/condition.sim index 56706467f1..c3aed7e2a3 100644 --- a/tests/script/general/parser/condition.sim +++ b/tests/script/general/parser/condition.sim @@ -3,6 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 +system sh/cfg.sh -n dnode1 -c cache -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -93,6 +94,47 @@ sql insert into tb3_2 values ('2021-05-06 18:19:28',15,NULL,15,NULL,15,NULL,true sql insert into tb3_2 values ('2021-06-06 18:19:28',NULL,16.0,NULL,16,NULL,16.0,NULL,'16',NULL) sql insert into tb3_2 values ('2021-07-06 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) + +sql create table stb4 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9),c10 binary(16300)) TAGS(t1 int, t2 binary(10), t3 double) + +sql create table tb4_0 using stb4 tags(0,'0',0.0) +sql create table tb4_1 using stb4 tags(1,'1',1.0) +sql create table tb4_2 using stb4 tags(2,'2',2.0) +sql create table tb4_3 using stb4 tags(3,'3',3.0) +sql create table tb4_4 using stb4 tags(4,'4',4.0) + +$i = 0 +$ts0 = 1625850000000 +$blockNum = 5 +$delta = 0 +$tbname0 = tb4_ +$a = 0 +$b = 200 +$c = 400 +while $i < $blockNum + $x = 0 + $rowNum = 1200 + while $x < $rowNum + $ts = $ts0 + $x + $a = $a + 1 + $b = $b + 1 + $c = $c + 1 + $d = $x / 10 + $tin = $rowNum + $binary = 'binary . $c + $binary = $binary . ' + $nchar = 'nchar . $c + $nchar = $nchar . ' + $tbname = 'tb4_ . $i + $tbname = $tbname . ' + sql insert into $tbname values ( $ts , $a , $b , $c , $d , $d , $c , true, $binary , $nchar , $binary ) + $x = $x + 1 + endw + + $i = $i + 1 + $ts0 = $ts0 + 259200000 +endw + sleep 100 sql connect diff --git a/tests/script/general/parser/condition_query.sim b/tests/script/general/parser/condition_query.sim index 7600e510d3..8dfa8dae0c 100644 --- a/tests/script/general/parser/condition_query.sim +++ b/tests/script/general/parser/condition_query.sim @@ -1959,6 +1959,117 @@ if $rows != 14 then return -1 endi +sql select ts,c1 from stb4 where c1 = 200; +if $rows != 1 then + return -1 +endi +if $data00 != @21-07-10 01:00:00.199@ then + return -1 +endi + +sql select ts,c1 from stb4 where c1 != 200; +if $rows != 5999 then + return -1 +endi + + + +sql select ts,c1,c2,c3,c4 from stb4 where c1 >= 200 and c2 > 500 and c3 < 800 and c4 between 33 and 37 and c4 != 35 and c2 < 555 and c1 < 339 and c1 in (331,333,335); +if $rows != 3 then + return -1 +endi +if $data00 != @21-07-10 01:00:00.330@ then + return -1 +endi +if $data10 != @21-07-10 01:00:00.332@ then + return -1 +endi +if $data20 != @21-07-10 01:00:00.334@ then + return -1 +endi + +sql select ts,c1,c2,c3,c4 from stb4 where c1 > -3 and c1 < 5; +if $rows != 4 then + return -1 +endi +if $data00 != @21-07-10 01:00:00.000@ then + return -1 +endi +if $data10 != @21-07-10 01:00:00.001@ then + return -1 +endi +if $data20 != @21-07-10 01:00:00.002@ then + return -1 +endi +if $data30 != @21-07-10 01:00:00.003@ then + return -1 +endi + +sql select ts,c1,c2,c3,c4 from stb4 where c1 >= 2 and c1 < 5; +if $rows != 3 then + return -1 +endi +if $data00 != @21-07-10 01:00:00.001@ then + return -1 +endi +if $data10 != @21-07-10 01:00:00.002@ then + return -1 +endi +if $data20 != @21-07-10 01:00:00.003@ then + return -1 +endi + +sql select ts,c1,c2,c3,c4 from stb4 where c1 >= -3 and c1 < 1300; +if $rows != 1299 then + return -1 +endi + +sql select ts,c1,c2,c3,c4 from stb4 where c1 >= 1298 and c1 < 1300 or c2 > 210 and c2 < 213; +if $rows != 4 then + return -1 +endi +if $data00 != @21-07-10 01:00:00.010@ then + return -1 +endi +if $data10 != @21-07-10 01:00:00.011@ then + return -1 +endi +if $data20 != @21-07-13 01:00:00.097@ then + return -1 +endi +if $data30 != @21-07-13 01:00:00.098@ then + return -1 +endi + +sql select ts,c1,c2,c3,c4 from stb4 where c1 >= -3; +if $rows != 6000 then + return -1 +endi + +sql select ts,c1,c2,c3,c4 from stb4 where c1 < 1400; +if $rows != 1399 then + return -1 +endi + +sql select ts,c1,c2,c3,c4 from stb4 where c1 < 1100; +if $rows != 1099 then + return -1 +endi + + +sql select ts,c1,c2,c3,c4 from stb4 where c1 in(10,100, 1100,3300) and c1 != 10; +if $rows != 3 then + return -1 +endi +if $data00 != @21-07-10 01:00:00.099@ then + return -1 +endi +if $data10 != @21-07-10 01:00:01.099@ then + return -1 +endi +if $data20 != @21-07-16 01:00:00.899@ then + return -1 +endi print "ts test" From 0b6a2ffdd808af5bb8ac4e29e33d3ab0e04592ec Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 15 Aug 2021 12:52:40 +0800 Subject: [PATCH 67/75] Hotfix/sangshuduo/td 3197 fix taosdemo coverity scan (#7374) * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix coverity scan issues. check super tbl info pointer. * [TD-3197] : fix coverity scan issues. move sub tbl query thread join into loop * [TD-3197] : fix coverity scan issues. remove unused variable * [TD-3197] : fix coverity scan issues. use more secure random library * [TD-3197] : fix coverity scan issues. use strncpy for more safe * [TD-3197] : fix taosdemo coverity scan issue. replace arc4random with rand(). * [TD-3197] : fix coverity scan issues. check stb info pointer for start time * [TD-3197] : fix coverity scan issues. fix strcpy vulnerability * [TD-3197] : fix taosdemo coverity scan issue. modify taosdemoTest2. try to check database continously. * [TD-3197] : taosdemo coverity scan issues. * [TD-3197] : fix memory leak when parsing arguments. * [TD-3197] : fix cmake strip arguments. * [TD-3197] : taosdemo coverity scan. fix cmake string manipulation. * [TD-3197]: taosdemo coverity scan issue. configDir buffer overwrite. * [TD-3197]: coverity scan issue. taosdump argument validation. * [TD-3197]: taosdemo and taosdump coverity scan issues. * [TD-3197]: taosdemo coverity scan. append result buf to file. for develop branch. * exit if read sample file failed. * fix converity scan issue. * fix coverity scan issue. * fix coverity scan memory leak. * fix resource leak reported by coverity scan. * fix taosdemo coverity scan issue. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index f3270a22f2..18f5877e09 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5820,6 +5820,7 @@ static int32_t prepareStmtBindArrayByType( } else if (0 == strncasecmp(dataType, "INT", strlen("INT"))) { int32_t *bind_int = malloc(sizeof(int32_t)); + assert(bind_int); if (value) { *bind_int = atoi(value); @@ -5834,6 +5835,7 @@ static int32_t prepareStmtBindArrayByType( } else if (0 == strncasecmp(dataType, "BIGINT", strlen("BIGINT"))) { int64_t *bind_bigint = malloc(sizeof(int64_t)); + assert(bind_bigint); if (value) { *bind_bigint = atoll(value); @@ -5848,6 +5850,7 @@ static int32_t prepareStmtBindArrayByType( } else if (0 == strncasecmp(dataType, "FLOAT", strlen("FLOAT"))) { float *bind_float = malloc(sizeof(float)); + assert(bind_float); if (value) { *bind_float = (float)atof(value); @@ -5862,6 +5865,7 @@ static int32_t prepareStmtBindArrayByType( } else if (0 == strncasecmp(dataType, "DOUBLE", strlen("DOUBLE"))) { double *bind_double = malloc(sizeof(double)); + assert(bind_double); if (value) { *bind_double = atof(value); @@ -5876,6 +5880,7 @@ static int32_t prepareStmtBindArrayByType( } else if (0 == strncasecmp(dataType, "SMALLINT", strlen("SMALLINT"))) { int16_t *bind_smallint = malloc(sizeof(int16_t)); + assert(bind_smallint); if (value) { *bind_smallint = (int16_t)atoi(value); @@ -5890,6 +5895,7 @@ static int32_t prepareStmtBindArrayByType( } else if (0 == strncasecmp(dataType, "TINYINT", strlen("TINYINT"))) { int8_t *bind_tinyint = malloc(sizeof(int8_t)); + assert(bind_tinyint); if (value) { *bind_tinyint = (int8_t)atoi(value); @@ -5904,6 +5910,7 @@ static int32_t prepareStmtBindArrayByType( } else if (0 == strncasecmp(dataType, "BOOL", strlen("BOOL"))) { int8_t *bind_bool = malloc(sizeof(int8_t)); + assert(bind_bool); if (value) { if (strncasecmp(value, "true", 4)) { @@ -5923,6 +5930,7 @@ static int32_t prepareStmtBindArrayByType( } else if (0 == strncasecmp(dataType, "TIMESTAMP", strlen("TIMESTAMP"))) { int64_t *bind_ts2 = malloc(sizeof(int64_t)); + assert(bind_ts2); if (value) { if (strchr(value, ':') && strchr(value, '-')) { @@ -5937,6 +5945,7 @@ static int32_t prepareStmtBindArrayByType( if (TSDB_CODE_SUCCESS != taosParseTime( value, &tmpEpoch, strlen(value), timePrec, 0)) { + free(bind_ts2); errorPrint("Input %s, time format error!\n", value); return -1; } @@ -6261,7 +6270,7 @@ static int32_t prepareStbStmtBindTag( char *bindBuffer = calloc(1, DOUBLE_BUFF_LEN); // g_args.len_of_binary); if (bindBuffer == NULL) { errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n", - __func__, __LINE__, g_args.len_of_binary); + __func__, __LINE__, DOUBLE_BUFF_LEN); return -1; } @@ -6293,7 +6302,7 @@ static int32_t prepareStbStmtBindRand( char *bindBuffer = calloc(1, DOUBLE_BUFF_LEN); // g_args.len_of_binary); if (bindBuffer == NULL) { errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n", - __func__, __LINE__, g_args.len_of_binary); + __func__, __LINE__, DOUBLE_BUFF_LEN); return -1; } From 02f6f5e457673e1201c233ac7fd92ea838d90377 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 15 Aug 2021 12:53:34 +0800 Subject: [PATCH 68/75] Feature/sangshuduo/td 5844 cmdline parameters align (#7239) * [TD-5844]: make cmd line parameter similar. * fix test case align with taosdemo change. * fix windows stack overflow issue. * fix mac compile error. * fix taosdemo cmdline parameter in tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoInsert.py * fix windows compiler options. * make taos.exe use mysql style password input. * make taos shell and taosdump use mysql style password input. * determine scanf return value. --- cmake/define.inc | 2 +- cmake/env.inc | 14 +++++++++-- src/kit/shell/src/shellDarwin.c | 23 ++++++++++++++--- src/kit/shell/src/shellEngine.c | 42 ++++++++++++++++++-------------- src/kit/shell/src/shellLinux.c | 35 +++++++++++++++++++++++--- src/kit/shell/src/shellMain.c | 2 ++ src/kit/shell/src/shellWindows.c | 30 +++++++++++++++++------ src/kit/taosdemo/taosdemo.c | 6 +++-- src/kit/taosdump/taosdump.c | 29 ++++++++++++++++++---- 9 files changed, 139 insertions(+), 44 deletions(-) diff --git a/cmake/define.inc b/cmake/define.inc index 6c466fee02..9ee09c86b0 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -180,7 +180,7 @@ IF (TD_WINDOWS) ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE) SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) IF (NOT TD_GODLL) - SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd2220 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") + SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd5999 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") ENDIF () diff --git a/cmake/env.inc b/cmake/env.inc index 2ceaecc2d9..a173a19749 100755 --- a/cmake/env.inc +++ b/cmake/env.inc @@ -34,12 +34,22 @@ ENDIF () # # Set compiler options -SET(COMMON_C_FLAGS "${COMMON_FLAGS} -std=gnu99") +IF (TD_LINUX) + SET(COMMON_C_FLAGS "${COMMON_FLAGS} -std=gnu99") +ELSE () + SET(COMMON_C_FLAGS "${COMMON_FLAGS} ") +ENDIF () + SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${COMMON_C_FLAGS} ${DEBUG_FLAGS}") SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_C_FLAGS} ${RELEASE_FLAGS}") # Set c++ compiler options -SET(COMMON_CXX_FLAGS "${COMMON_FLAGS} -std=c++11 -Wno-unused-function") +IF (TD_WINDOWS) + SET(COMMON_CXX_FLAGS "${COMMON_FLAGS} -std=c++11") +ELSE () + SET(COMMON_CXX_FLAGS "${COMMON_FLAGS} -std=c++11 -Wno-unused-function") +ENDIF () + SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COMMON_CXX_FLAGS} ${DEBUG_FLAGS}") SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COMMON_CXX_FLAGS} ${RELEASE_FLAGS}") diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index 4dcd8b3d50..5ca4537aeb 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -64,6 +64,10 @@ void printHelp() { exit(EXIT_SUCCESS); } +char DARWINCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" + "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; +char g_password[MAX_PASSWORD_SIZE]; + void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { wordexp_t full_path; for (int i = 1; i < argc; i++) { @@ -77,10 +81,21 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { } } // for password - else if (strcmp(argv[i], "-p") == 0) { - arguments->is_use_passwd = true; + else if (strncmp(argv[i], "-p", 2) == 0) { + strcpy(tsOsName, "Darwin"); + printf(DARWINCLIENT_VERSION, tsOsName, taos_get_client_info()); + if (strlen(argv[i]) == 2) { + printf("Enter password: "); + if (scanf("%s", g_password) > 1) { + fprintf(stderr, "password read error\n"); + } + getchar(); + } else { + tstrncpy(g_password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE); + } + arguments->password = g_password; } - // for management port + // for management port else if (strcmp(argv[i], "-P") == 0) { if (i < argc - 1) { arguments->port = atoi(argv[++i]); @@ -98,7 +113,7 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { exit(EXIT_FAILURE); } } else if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { + if (i < argc - 1) { if (strlen(argv[++i]) >= TSDB_FILENAME_LEN) { fprintf(stderr, "config file path: %s overflow max len %d\n", argv[i], TSDB_FILENAME_LEN - 1); exit(EXIT_FAILURE); diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index cdce61e578..bf19394d05 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -65,7 +65,15 @@ extern TAOS *taos_connect_auth(const char *ip, const char *user, const char *aut */ TAOS *shellInit(SShellArguments *_args) { printf("\n"); - printf(CLIENT_VERSION, tsOsName, taos_get_client_info()); + if (!_args->is_use_passwd) { +#ifdef TD_WINDOWS + strcpy(tsOsName, "Windows"); +#elif defined(TD_DARWIN) + strcpy(tsOsName, "Darwin"); +#endif + printf(CLIENT_VERSION, tsOsName, taos_get_client_info()); + } + fflush(stdout); // set options before initializing @@ -73,9 +81,7 @@ TAOS *shellInit(SShellArguments *_args) { taos_options(TSDB_OPTION_TIMEZONE, _args->timezone); } - if (_args->is_use_passwd) { - if (_args->password == NULL) _args->password = getpass("Enter password: "); - } else { + if (!_args->is_use_passwd) { _args->password = TSDB_DEFAULT_PASS; } @@ -169,7 +175,7 @@ static int32_t shellRunSingleCommand(TAOS *con, char *command) { system("clear"); return 0; } - + if (regex_match(command, "^[\t ]*set[ \t]+max_binary_display_width[ \t]+(default|[1-9][0-9]*)[ \t;]*$", REG_EXTENDED | REG_ICASE)) { strtok(command, " \t"); strtok(NULL, " \t"); @@ -181,7 +187,7 @@ static int32_t shellRunSingleCommand(TAOS *con, char *command) { } return 0; } - + if (regex_match(command, "^[ \t]*source[\t ]+[^ ]+[ \t;]*$", REG_EXTENDED | REG_ICASE)) { /* If source file. */ char *c_ptr = strtok(command, " ;"); @@ -246,7 +252,7 @@ int32_t shellRunCommand(TAOS* con, char* command) { esc = false; continue; } - + if (c == '\\') { esc = true; continue; @@ -335,8 +341,8 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { } if (!tscIsUpdateQuery(pSql)) { // select and show kinds of commands - int error_no = 0; - + int error_no = 0; + int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); if (numOfRows < 0) { atomic_store_64(&result, 0); @@ -529,7 +535,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { fprintf(fp, "%s", fields[col].name); } fputc('\n', fp); - + int numOfRows = 0; do { int32_t* length = taos_fetch_lengths(tres); @@ -715,7 +721,7 @@ static int verticalPrintResult(TAOS_RES* tres) { int numOfRows = 0; int showMore = 1; - do { + do { if (numOfRows < resShowMaxNum) { printf("*************************** %d.row ***************************\n", numOfRows + 1); @@ -850,7 +856,7 @@ static int horizontalPrintResult(TAOS_RES* tres) { int numOfRows = 0; int showMore = 1; - + do { int32_t* length = taos_fetch_lengths(tres); if (numOfRows < resShowMaxNum) { @@ -866,7 +872,7 @@ static int horizontalPrintResult(TAOS_RES* tres) { printf("[You can add limit statement to show more or redirect results to specific file to get all.]\n"); showMore = 0; } - + numOfRows++; row = taos_fetch_row(tres); } while(row != NULL); @@ -908,7 +914,7 @@ void read_history() { if (errno != ENOENT) { fprintf(stderr, "Failed to open file %s, reason:%s\n", f_history, strerror(errno)); } -#endif +#endif return; } @@ -933,9 +939,9 @@ void write_history() { FILE *f = fopen(f_history, "w"); if (f == NULL) { -#ifndef WINDOWS +#ifndef WINDOWS fprintf(stderr, "Failed to open file %s for write, reason:%s\n", f_history, strerror(errno)); -#endif +#endif return; } @@ -981,13 +987,13 @@ void source_file(TAOS *con, char *fptr) { /* if (access(fname, F_OK) != 0) { fprintf(stderr, "ERROR: file %s is not exist\n", fptr); - + wordfree(&full_path); free(cmd); return; } */ - + FILE *f = fopen(fname, "r"); if (f == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index dc74f6fcaa..d051d3535e 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -34,7 +34,7 @@ static char doc[] = ""; static char args_doc[] = ""; static struct argp_option options[] = { {"host", 'h', "HOST", 0, "TDengine server FQDN to connect. The default host is localhost."}, - {"password", 'p', "PASSWORD", OPTION_ARG_OPTIONAL, "The password to use when connecting to the server."}, + {"password", 'p', 0, 0, "The password to use when connecting to the server."}, {"port", 'P', "PORT", 0, "The TCP/IP port number to use for the connection."}, {"user", 'u', "USER", 0, "The user name to use when connecting to the server."}, {"auth", 'A', "Auth", 0, "The auth string to use when connecting to the server."}, @@ -63,8 +63,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { arguments->host = arg; break; case 'p': - arguments->is_use_passwd = true; - if (arg) arguments->password = arg; break; case 'P': if (arg) { @@ -160,12 +158,41 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { /* Our argp parser. */ static struct argp argp = {options, parse_opt, args_doc, doc}; +char LINUXCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" + "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; +char g_password[MAX_PASSWORD_SIZE]; + +static void parse_password( + int argc, char *argv[], SShellArguments *arguments) { + for (int i = 1; i < argc; i++) { + if (strncmp(argv[i], "-p", 2) == 0) { + strcpy(tsOsName, "Linux"); + printf(LINUXCLIENT_VERSION, tsOsName, taos_get_client_info()); + if (strlen(argv[i]) == 2) { + printf("Enter password: "); + if (scanf("%20s", g_password) > 1) { + fprintf(stderr, "password reading error\n"); + } + getchar(); + } else { + tstrncpy(g_password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE); + } + arguments->password = g_password; + arguments->is_use_passwd = true; + } + } +} + void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { static char verType[32] = {0}; sprintf(verType, "version: %s\n", version); argp_program_version = verType; - + + if (argc > 1) { + parse_password(argc, argv, arguments); + } + argp_parse(&argp, argc, argv, 0, 0, arguments); if (arguments->abort) { #ifndef _ALPINE diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 0c70386061..4e00b0d8ff 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -71,7 +71,9 @@ int checkVersion() { // Global configurations SShellArguments args = { .host = NULL, +#ifndef TD_WINDOWS .password = NULL, +#endif .user = NULL, .database = NULL, .timezone = NULL, diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index 87d11a3516..bf9afe4b80 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -19,6 +19,9 @@ extern char configDir[]; +char WINCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" + "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; + void printVersion() { printf("version: %s\n", version); } @@ -61,6 +64,8 @@ void printHelp() { exit(EXIT_SUCCESS); } +char g_password[MAX_PASSWORD_SIZE]; + void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { for (int i = 1; i < argc; i++) { // for host @@ -73,11 +78,20 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { } } // for password - else if (strcmp(argv[i], "-p") == 0) { - arguments->is_use_passwd = true; - if (i < argc - 1 && argv[i + 1][0] != '-') { - arguments->password = argv[++i]; - } + else if (strncmp(argv[i], "-p", 2) == 0) { + arguments->is_use_passwd = true; + strcpy(tsOsName, "Windows"); + printf(WINCLIENT_VERSION, tsOsName, taos_get_client_info()); + if (strlen(argv[i]) == 2) { + printf("Enter password: "); + if (scanf("%s", g_password) > 1) { + fprintf(stderr, "password read error!\n"); + } + getchar(); + } else { + tstrncpy(g_password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE); + } + arguments->password = g_password; } // for management port else if (strcmp(argv[i], "-P") == 0) { @@ -104,7 +118,7 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { exit(EXIT_FAILURE); } } else if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { + if (i < argc - 1) { char *tmp = argv[++i]; if (strlen(tmp) >= TSDB_FILENAME_LEN) { fprintf(stderr, "config file path: %s overflow max len %d\n", tmp, TSDB_FILENAME_LEN - 1); @@ -265,7 +279,7 @@ void *shellLoopQuery(void *arg) { if (command == NULL) return NULL; int32_t err = 0; - + do { memset(command, 0, MAX_COMMAND_SIZE); shellPrintPrompt(); @@ -274,7 +288,7 @@ void *shellLoopQuery(void *arg) { err = shellReadCommand(con, command); if (err) { break; - } + } } while (shellRunCommand(con, command) == 0); return NULL; diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 18f5877e09..07dbc5e022 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -77,7 +77,7 @@ extern char configDir[]; #define COL_BUFFER_LEN ((TSDB_COL_NAME_LEN + 15) * TSDB_MAX_COLUMNS) #define MAX_USERNAME_SIZE 64 -#define MAX_PASSWORD_SIZE 16 +#define MAX_PASSWORD_SIZE 20 #define MAX_HOSTNAME_SIZE 253 // https://man7.org/linux/man-pages/man7/hostname.7.html #define MAX_TB_NAME_SIZE 64 #define MAX_DATA_SIZE (16*TSDB_MAX_COLUMNS)+20 // max record len: 16*MAX_COLUMNS, timestamp string and ,('') need extra space @@ -867,7 +867,9 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } else if (strncmp(argv[i], "-p", 2) == 0) { if (strlen(argv[i]) == 2) { printf("Enter password:"); - scanf("%s", arguments->password); + if (scanf("%s", arguments->password) > 1) { + fprintf(stderr, "password read error!\n"); + } } else { tstrncpy(arguments->password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE); } diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index bea6e65106..b7073c28a7 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -206,9 +206,9 @@ static struct argp_option options[] = { {"host", 'h', "HOST", 0, "Server host dumping data from. Default is localhost.", 0}, {"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0}, #ifdef _TD_POWER_ - {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is powerdb.", 0}, + {"password", 'p', 0, 0, "User password to connect to server. Default is powerdb.", 0}, #else - {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is taosdata.", 0}, + {"password", 'p', 0, 0, "User password to connect to server. Default is taosdata.", 0}, #endif {"port", 'P', "PORT", 0, "Port to connect", 0}, {"cversion", 'v', "CVERION", 0, "client version", 0}, @@ -248,12 +248,14 @@ static struct argp_option options[] = { {0} }; +#define MAX_PASSWORD_SIZE 20 + /* Used by main to communicate with parse_opt. */ typedef struct arguments { // connection option char *host; char *user; - char *password; + char password[MAX_PASSWORD_SIZE]; uint16_t port; char cversion[12]; uint16_t mysqlFlag; @@ -376,7 +378,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { g_args.user = arg; break; case 'p': - g_args.password = arg; break; case 'P': g_args.port = atoi(arg); @@ -554,6 +555,23 @@ static void parse_precision_first( } } +static void parse_password( + int argc, char *argv[], SArguments *arguments) { + for (int i = 1; i < argc; i++) { + if (strncmp(argv[i], "-p", 2) == 0) { + if (strlen(argv[i]) == 2) { + printf("Enter password: "); + if(scanf("%20s", arguments->password) > 1) { + errorPrint("%s() LN%d, password read error!\n", __func__, __LINE__); + } + } else { + tstrncpy(arguments->password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE); + } + argv[i] = ""; + } + } +} + static void parse_timestamp( int argc, char *argv[], SArguments *arguments) { for (int i = 1; i < argc; i++) { @@ -616,9 +634,10 @@ int main(int argc, char *argv[]) { int ret = 0; /* Parse our arguments; every option seen by parse_opt will be reflected in arguments. */ - if (argc > 2) { + if (argc > 1) { parse_precision_first(argc, argv, &g_args); parse_timestamp(argc, argv, &g_args); + parse_password(argc, argv, &g_args); } argp_parse(&argp, argc, argv, 0, 0, &g_args); From 37a8e1bad15e924b07022d8b7104d9db3bd8444a Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Sun, 15 Aug 2021 19:18:12 +0800 Subject: [PATCH 69/75] [TD-4181] : update Connections related doc. --- documentation20/cn/09.connections/docs.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/documentation20/cn/09.connections/docs.md b/documentation20/cn/09.connections/docs.md index 7a13270830..7c70724823 100644 --- a/documentation20/cn/09.connections/docs.md +++ b/documentation20/cn/09.connections/docs.md @@ -23,7 +23,7 @@ sudo cp -rf /usr/local/taos/connector/grafanaplugin /var/lib/grafana/plugins/tde #### 配置数据源 -用户可以直接通过 localhost:3000 的网址，登录 Grafana 服务器(用户名/密码:admin/admin)，通过左侧 `Configuration -> Data Sources` 可以添加数据源，如下图所示： +用户可以直接通过 localhost:3000 的网址，登录 Grafana 服务器（用户名/密码：admin/admin），通过左侧 `Configuration -> Data Sources` 可以添加数据源，如下图所示： ![img](page://images/connections/add_datasource1.jpg) @@ -35,7 +35,7 @@ sudo cp -rf /usr/local/taos/connector/grafanaplugin /var/lib/grafana/plugins/tde ![img](page://images/connections/add_datasource3.jpg) -* Host： TDengine 集群的中任意一台服务器的 IP 地址与 TDengine RESTful 接口的端口号(6041)，默认 http://localhost:6041 +* Host： TDengine 集群的中任意一台服务器的 IP 地址与 TDengine RESTful 接口的端口号(6041)，默认 http://localhost:6041 。 * User：TDengine 用户名。 * Password：TDengine 用户密码。 @@ -140,13 +140,13 @@ conn<-dbConnect(drv,"jdbc:TSDB://192.168.0.1:0/?user=root&password=taosdata","ro - dbWriteTable(conn, "test", iris, overwrite=FALSE, append=TRUE)：将数据框iris写入表test中，overwrite必须设置为false，append必须设为TRUE,且数据框iris要与表test的结构一致。 -- dbGetQuery(conn, "select count(*) from test")：查询语句 +- dbGetQuery(conn, "select count(*) from test")：查询语句。 - dbSendUpdate(conn, "use db")：执行任何非查询sql语句。例如dbSendUpdate(conn, "use db")， 写入数据dbSendUpdate(conn, "insert into t1 values(now, 99)")等。 -- dbReadTable(conn, "test")：读取表test中数据 -- dbDisconnect(conn)：关闭连接 -- dbRemoveTable(conn, "test")：删除表test +- dbReadTable(conn, "test")：读取表test中数据。 +- dbDisconnect(conn)：关闭连接。 +- dbRemoveTable(conn, "test")：删除表test。 TDengine客户端暂不支持如下函数： -- dbExistsTable(conn, "test")：是否存在表test -- dbListTables(conn)：显示连接中的所有表 +- dbExistsTable(conn, "test")：是否存在表test。 +- dbListTables(conn)：显示连接中的所有表。 From a68e247cd138b28bce0b941a2a49af1b05e1ff56 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Sun, 15 Aug 2021 21:36:31 +0800 Subject: [PATCH 70/75] [TD-4181] : update Connector related doc. --- .../cn/08.connector/01.java/docs.md | 197 ++++++++------- documentation20/cn/08.connector/docs.md | 239 +++++++++++++----- 2 files changed, 286 insertions(+), 150 deletions(-) diff --git a/documentation20/cn/08.connector/01.java/docs.md b/documentation20/cn/08.connector/01.java/docs.md index 511bab8a60..f22a1a3803 100644 --- a/documentation20/cn/08.connector/01.java/docs.md +++ b/documentation20/cn/08.connector/01.java/docs.md @@ -1,6 +1,72 @@ # Java Connector -TDengine 提供了遵循 JDBC 标准（3.0）API 规范的 `taos-jdbcdriver` 实现，可在 maven 的中央仓库 [Sonatype Repository][1] 搜索下载。 +## 安装 + +Java连接器支持的系统有： Linux 64/Windows x64/Windows x86。 + +**安装前准备：** + +- 已安装TDengine服务器端 +- 已安装好TDengine应用驱动，具体请参照 [安装连接器驱动步骤](https://www.taosdata.com/cn/documentation/connector#driver) 章节 + +TDengine 为了方便 Java 应用使用，提供了遵循 JDBC 标准(3.0)API 规范的 `taos-jdbcdriver` 实现。目前可以通过 [Sonatype Repository](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) 搜索并下载。 + +由于 TDengine 的应用驱动是使用C语言开发的，使用 taos-jdbcdriver 驱动包时需要依赖系统对应的本地函数库。 + +- libtaos.so 在 Linux 系统中成功安装 TDengine 后，依赖的本地函数库 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so，该目录包含在 Linux 自动扫描路径上，无需单独指定。 + +- taos.dll 在 Windows 系统中安装完客户端之后，驱动包依赖的 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下，同样无需要单独指定。 + +注意：在 Windows 环境开发时需要安装 TDengine 对应的 [windows 客户端](https://www.taosdata.com/cn/all-downloads/#TDengine-Windows-Client)，Linux 服务器安装完 TDengine 之后默认已安装 client，也可以单独安装 [Linux 客户端](https://www.taosdata.com/cn/getting-started/#快速上手) 连接远程 TDengine Server。 + +### 如何获取 TAOS-JDBCDriver + +**maven仓库** + +目前 taos-jdbcdriver 已经发布到 [Sonatype Repository](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) 仓库，且各大仓库都已同步。 + +- [sonatype](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) +- [mvnrepository](https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver) +- [maven.aliyun](https://maven.aliyun.com/mvn/search) + +maven 项目中使用如下 pom.xml 配置即可： +```xml-dtd + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.18 + +``` +**源码编译打包** + +下载 TDengine 源码之后，进入 taos-jdbcdriver 源码目录 `src/connector/jdbc` 执行 `mvn clean package -Dmaven.test.skip=true` 即可生成相应 jar 包。 + +### 示例程序 + +示例程序源码位于install_directory/examples/JDBC，有如下目录： + +JDBCDemo JDBC示例源程序 + +JDBCConnectorChecker JDBC安装校验源程序及jar包 + +Springbootdemo springboot示例源程序 + +SpringJdbcTemplate SpringJDBC模板 + + +### 安装验证 + +运行如下指令： + +```Bash +cd {install_directory}/examples/JDBC/JDBCConnectorChecker +java -jar JDBCConnectorChecker.jar -host +``` + +验证通过将打印出成功信息。 + + +## Java连接器的使用 `taos-jdbcdriver` 的实现包括 2 种形式： JDBC-JNI 和 JDBC-RESTful（taos-jdbcdriver-2.0.18 开始支持 JDBC-RESTful）。 JDBC-JNI 通过调用客户端 libtaos.so（或 taos.dll ）的本地方法实现， JDBC-RESTful 则在内部封装了 RESTful 接口实现。 @@ -20,7 +86,7 @@ TDengine 的 JDBC 驱动实现尽可能与关系型数据库驱动保持一致 * 对每个 Connection 的实例，至多只能有一个打开的 ResultSet 实例；如果在 ResultSet 还没关闭的情况下执行了新的查询，taos-jdbcdriver 会自动关闭上一个 ResultSet。 -## JDBC-JNI和JDBC-RESTful的对比 +### JDBC-JNI和JDBC-RESTful的对比 @@ -51,33 +117,34 @@ TDengine 的 JDBC 驱动实现尽可能与关系型数据库驱动保持一致 注意：与 JNI 方式不同，RESTful 接口是无状态的，因此 `USE db_name` 指令没有效果，RESTful 下所有对表名、超级表名的引用都需要指定数据库名前缀。 -## 如何获取 taos-jdbcdriver +### TAOS-JDBCDriver 版本以及支持的 TDengine 版本和 JDK 版本 -### maven 仓库 +| taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 | +| -------------------- | ----------------- | -------- | +| 2.0.31 | 2.1.3.0 及以上 | 1.8.x | +| 2.0.22 - 2.0.30 | 2.0.18.0 - 2.1.2.x | 1.8.x | +| 2.0.12 - 2.0.21 | 2.0.8.0 - 2.0.17.x | 1.8.x | +| 2.0.4 - 2.0.11 | 2.0.0.0 - 2.0.7.x | 1.8.x | +| 1.0.3 | 1.6.1.x 及以上 | 1.8.x | +| 1.0.2 | 1.6.1.x 及以上 | 1.8.x | +| 1.0.1 | 1.6.1.x 及以上 | 1.8.x | -目前 taos-jdbcdriver 已经发布到 [Sonatype Repository][1] 仓库，且各大仓库都已同步。 +### TDengine DataType 和 Java DataType -* [sonatype][8] -* [mvnrepository][9] -* [maven.aliyun][10] +TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对应类型转换如下： -maven 项目中使用如下 pom.xml 配置即可： - -```xml - - com.taosdata.jdbc - taos-jdbcdriver - 2.0.18 - -``` - -### 源码编译打包 - -下载 [TDengine][3] 源码之后，进入 taos-jdbcdriver 源码目录 `src/connector/jdbc` 执行 `mvn clean package -Dmaven.test.skip=true` 即可生成相应 jar 包。 - - - -## JDBC的使用说明 +| TDengine DataType | Java DataType | +| ----------------- | ------------------ | +| TIMESTAMP | java.sql.Timestamp | +| INT | java.lang.Integer | +| BIGINT | java.lang.Long | +| FLOAT | java.lang.Float | +| DOUBLE | java.lang.Double | +| SMALLINT | java.lang.Short | +| TINYINT | java.lang.Byte | +| BOOL | java.lang.Boolean | +| BINARY | byte array | +| NCHAR | java.lang.String | ### 获取连接 @@ -112,12 +179,12 @@ Connection conn = DriverManager.getConnection(jdbcUrl); **注意**：使用 JDBC-JNI 的 driver，taos-jdbcdriver 驱动包时需要依赖系统对应的本地函数库。 * libtaos.so - 在 linux 系统中成功安装 TDengine 后，依赖的本地函数库 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so，该目录包含在 Linux 自动扫描路径上，无需单独指定。 + 在 Linux 系统中成功安装 TDengine 后，依赖的本地函数库 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so，该目录包含在 Linux 自动扫描路径上，无需单独指定。 * taos.dll - 在 windows 系统中安装完客户端之后，驱动包依赖的 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下，同样无需要单独指定。 + 在 Windows 系统中安装完客户端之后，驱动包依赖的 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下，同样无需要单独指定。 -> 在 windows 环境开发时需要安装 TDengine 对应的 [windows 客户端][14]，Linux 服务器安装完 TDengine 之后默认已安装 client，也可以单独安装 [Linux 客户端][15] 连接远程 TDengine Server。 +> 在 Windows 环境开发时需要安装 TDengine 对应的 [windows 客户端][14]，Linux 服务器安装完 TDengine 之后默认已安装 client，也可以单独安装 [Linux 客户端][15] 连接远程 TDengine Server。 JDBC-JNI 的使用请参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1955.html)。 @@ -166,8 +233,7 @@ properties 中的配置参数如下： #### 使用客户端配置文件建立连接 -当使用 JDBC-JNI 连接 TDengine 集群时，可以使用客户端配置文件，在客户端配置文件中指定集群的 firstEp、secondEp参数。 -如下所示： +当使用 JDBC-JNI 连接 TDengine 集群时，可以使用客户端配置文件，在客户端配置文件中指定集群的 firstEp、secondEp参数。如下所示： 1. 在 Java 应用中不指定 hostname 和 port @@ -214,7 +280,7 @@ TDengine 中，只要保证 firstEp 和 secondEp 中一个节点有效，就可 例如：在 url 中指定了 password 为 taosdata，在 Properties 中指定了 password 为 taosdemo，那么，JDBC 会使用 url 中的 password 建立连接。 -> 更多详细配置请参考[客户端配置][13] +> 更多详细配置请参考[客户端配置](https://www.taosdata.com/cn/documentation/administrator/#client) ### 创建数据库和表 @@ -242,8 +308,8 @@ int affectedRows = stmt.executeUpdate("insert into tb values(now, 23, 10.3) (now System.out.println("insert " + affectedRows + " rows."); ``` -> now 为系统内部函数，默认为服务器当前时间。 -> `now + 1s` 代表服务器当前时间往后加 1 秒，数字后面代表时间单位：a(毫秒), s(秒), m(分), h(小时), d(天)，w(周), n(月), y(年)。 +> now 为系统内部函数，默认为客户端所在计算机当前时间。 +> `now + 1s` 代表客户端当前时间往后加 1 秒，数字后面代表时间单位：a(毫秒)，s(秒)，m(分)，h(小时)，d(天)，w(周)，n(月)，y(年)。 ### 查询数据 @@ -464,7 +530,7 @@ conn.close(); ``` > 通过 HikariDataSource.getConnection() 获取连接后，使用完成后需要调用 close() 方法，实际上它并不会关闭连接，只是放回连接池中。 -> 更多 HikariCP 使用问题请查看[官方说明][5] +> 更多 HikariCP 使用问题请查看[官方说明](https://github.com/brettwooldridge/HikariCP) **Druid** @@ -505,13 +571,13 @@ public static void main(String[] args) throws Exception { } ``` -> 更多 druid 使用问题请查看[官方说明][6] +> 更多 druid 使用问题请查看[官方说明](https://github.com/alibaba/druid) **注意事项** * TDengine `v1.6.4.1` 版本开始提供了一个专门用于心跳检测的函数 `select server_status()`，所以在使用连接池时推荐使用 `select server_status()` 进行 Validation Query。 如下所示，`select server_status()` 执行成功会返回 `1`。 -```shell +```sql taos> select server_status(); server_status()| ================ @@ -521,43 +587,10 @@ Query OK, 1 row(s) in set (0.000141s) -## 与框架使用 +## 在框架中使用 -* Spring JdbcTemplate 中使用 taos-jdbcdriver，可参考 [SpringJdbcTemplate][11] -* Springboot + Mybatis 中使用，可参考 [springbootdemo][12] - - - -## TAOS-JDBCDriver 版本以及支持的 TDengine 版本和 JDK 版本 - -| taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 | -| -------------------- | ----------------- | -------- | -| 2.0.31 | 2.1.3.0 及以上 | 1.8.x | -| 2.0.22 - 2.0.30 | 2.0.18.0 - 2.1.2.x | 1.8.x | -| 2.0.12 - 2.0.21 | 2.0.8.0 - 2.0.17.x | 1.8.x | -| 2.0.4 - 2.0.11 | 2.0.0.0 - 2.0.7.x | 1.8.x | -| 1.0.3 | 1.6.1.x 及以上 | 1.8.x | -| 1.0.2 | 1.6.1.x 及以上 | 1.8.x | -| 1.0.1 | 1.6.1.x 及以上 | 1.8.x | - - - -## TDengine DataType 和 Java DataType - -TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对应类型转换如下： - -| TDengine DataType | Java DataType | -| ----------------- | ------------------ | -| TIMESTAMP | java.sql.Timestamp | -| INT | java.lang.Integer | -| BIGINT | java.lang.Long | -| FLOAT | java.lang.Float | -| DOUBLE | java.lang.Double | -| SMALLINT | java.lang.Short | -| TINYINT | java.lang.Byte | -| BOOL | java.lang.Boolean | -| BINARY | byte array | -| NCHAR | java.lang.String | +* Spring JdbcTemplate 中使用 taos-jdbcdriver，可参考 [SpringJdbcTemplate](https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/SpringJdbcTemplate) +* Springboot + Mybatis 中使用，可参考 [springbootdemo](https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/springbootdemo) @@ -567,7 +600,7 @@ TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对 **原因**：程序没有找到依赖的本地函数库 taos。 - **解决方法**：windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下，linux 下将建立如下软链 `ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可。 + **解决方法**：Windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下，Linux 下将建立如下软链 `ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可。 * java.lang.UnsatisfiedLinkError: taos.dll Can't load AMD 64 bit on a IA 32-bit platform @@ -575,21 +608,5 @@ TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对 **解决方法**：重新安装 64 位 JDK。 -* 其它问题请参考 [Issues][7] - -[1]: https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver -[2]: https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver -[3]: https://github.com/taosdata/TDengine -[4]: https://www.taosdata.com/blog/2019/12/03/jdbcdriver%e6%89%be%e4%b8%8d%e5%88%b0%e5%8a%a8%e6%80%81%e9%93%be%e6%8e%a5%e5%ba%93/ -[5]: https://github.com/brettwooldridge/HikariCP -[6]: https://github.com/alibaba/druid -[7]: https://github.com/taosdata/TDengine/issues -[8]: https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver -[9]: https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver -[10]: https://maven.aliyun.com/mvn/search -[11]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/SpringJdbcTemplate -[12]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/springbootdemo -[13]: https://www.taosdata.com/cn/documentation/administrator/#client -[14]: https://www.taosdata.com/cn/all-downloads/#TDengine-Windows-Client -[15]: https://www.taosdata.com/cn/getting-started/#%E5%AE%A2%E6%88%B7%E7%AB%AF +* 其它问题请参考 [Issues](https://github.com/taosdata/TDengine/issues) diff --git a/documentation20/cn/08.connector/docs.md b/documentation20/cn/08.connector/docs.md index 90bcc51193..72fd89c274 100644 --- a/documentation20/cn/08.connector/docs.md +++ b/documentation20/cn/08.connector/docs.md @@ -23,7 +23,7 @@ TDengine提供了丰富的应用程序开发接口，其中包括C/C++、Java、 * 在没有安装TDengine服务端软件的系统中使用连接器（除RESTful外）访问 TDengine 数据库，需要安装相应版本的客户端安装包来使应用驱动（Linux系统中文件名为libtaos.so，Windows系统中为taos.dll）被安装在系统中，否则会产生无法找到相应库文件的错误。 * 所有执行 SQL 语句的 API，例如 C/C++ Connector 中的 `tao_query`、`taos_query_a`、`taos_subscribe` 等，以及其它语言中与它们对应的API，每次都只能执行一条 SQL 语句，如果实际参数中包含了多条语句，它们的行为是未定义的。 -* 升级到TDengine到2.0.8.0版本的用户，必须更新JDBC连接TDengine必须升级taos-jdbcdriver到2.0.12及以上。详细的版本依赖关系请参见 [taos-jdbcdriver 文档](https://www.taosdata.com/cn/documentation/connector/java#version)。 +* 升级 TDengine 到 2.0.8.0 版本的用户，必须更新 JDBC。连接 TDengine 必须升级 taos-jdbcdriver 到 2.0.12 及以上。详细的版本依赖关系请参见 [taos-jdbcdriver 文档](https://www.taosdata.com/cn/documentation/connector/java#version)。 * 无论选用何种编程语言的连接器，2.0 及以上版本的 TDengine 推荐数据库应用的每个线程都建立一个独立的连接，或基于线程建立连接池，以避免连接内的“USE statement”状态量在线程之间相互干扰（但连接的查询和写入操作都是线程安全的）。 ## 安装连接器驱动步骤 @@ -32,7 +32,7 @@ TDengine提供了丰富的应用程序开发接口，其中包括C/C++、Java、 **Linux** -**1. 从[涛思官网](https://www.taosdata.com/cn/all-downloads/)下载** +**1. 从[涛思官网](https://www.taosdata.com/cn/all-downloads/)下载：** * X64硬件环境：TDengine-client-2.x.x.x-Linux-x64.tar.gz @@ -46,7 +46,7 @@ TDengine提供了丰富的应用程序开发接口，其中包括C/C++、Java、 `tar -xzvf TDengine-client-xxxxxxxxx.tar.gz` -其中xxxxxxx需要替换为实际版本的字符串。 +其中xxxxxxxxx需要替换为实际版本的字符串。 **3. 执行安装脚本** @@ -68,7 +68,7 @@ TDengine提供了丰富的应用程序开发接口，其中包括C/C++、Java、 **Windows x64/x86** -**1. 从[涛思官网](https://www.taosdata.com/cn/all-downloads/)下载 ：** +**1. 从[涛思官网](https://www.taosdata.com/cn/all-downloads/)下载：** * X64硬件环境：TDengine-client-2.X.X.X-Windows-x64.exe @@ -99,13 +99,13 @@ TDengine提供了丰富的应用程序开发接口，其中包括C/C++、Java、 **2．卸载：运行unins000.exe可卸载TDengine应用驱动。** -**安装验证** +### 安装验证 以上安装和配置完成后，并确认TDengine服务已经正常启动运行，此时可以执行taos客户端进行登录。 **Linux环境：** -在linux shell下直接执行 taos，应该就能正常链接到tdegine服务，进入到taos shell界面，示例如下： +在Linux shell下直接执行 taos，应该就能正常连接到TDegine服务，进入到taos shell界面，示例如下： ```mysql $ taos @@ -146,7 +146,10 @@ taos> | **OS类型** | Linux | Win64 | Win32 | Linux | Linux | | **支持与否** | **支持** | **支持** | **支持** | **支持** | **开发中** | -C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine头文件 _taos.h_（安装后，位于 _/usr/local/taos/include_）： +C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine头文件 *taos.h*，里面列出了提供的API的函数原型。安装后，taos.h位于： + +- Linux：`/usr/local/taos/include` +- Windows：`C:\TDengine\include` ```C #include @@ -156,9 +159,22 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine * 在编译时需要链接TDengine动态库。Linux 为 *libtaos.so* ，安装后，位于 _/usr/local/taos/driver_。Windows为 taos.dll，安装后位于 *C:\TDengine*。 * 如未特别说明，当API的返回值是整数时，_0_ 代表成功，其它是代表失败原因的错误码，当返回值是指针时， _NULL_ 表示失败。 +* 在 taoserror.h中有所有的错误码，以及对应的原因描述。 + +### 示例程序 使用C/C++连接器的示例代码请参见 https://github.com/taosdata/TDengine/tree/develop/tests/examples/c 。 +示例程序源码也可以在安装目录下的 examples/c 路径下找到： + +**apitest.c、asyncdemo.c、demo.c、prepare.c、stream.c、subscribe.c** + +该目录下有makefile，在Linux环境下，直接执行make就可以编译得到执行文件。 + +在一台机器上启动TDengine服务，执行这些示例程序，按照提示输入TDengine服务的FQDN，就可以正常运行，并打印出信息。 + +**提示：**在ARM环境下编译时，请将makefile中的-msse4.2打开，这个选项只有在x64/x86硬件平台上才能支持。 + ### 基础API 基础API用于完成创建数据库连接等工作，为其它API的执行提供运行时环境。 @@ -187,7 +203,7 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine - user：用户名 - pass：密码 - db：数据库名字，如果用户没有提供，也可以正常连接，用户可以通过该连接创建新的数据库，如果用户提供了数据库名字，则说明该数据库用户已经创建好，缺省使用该数据库 - - port：端口号 + - port：TDengine管理主节点的端口号 返回值为空表示失败。应用程序需要保存返回的参数，以便后续API调用。 @@ -201,7 +217,7 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine - `void taos_close(TAOS *taos)` - 关闭连接, 其中`taos`是`taos_connect`函数返回的指针。 + 关闭连接，其中`taos`是`taos_connect`函数返回的指针。 ### 同步查询API @@ -237,13 +253,13 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine - `TAOS_FIELD *taos_fetch_fields(TAOS_RES *res)` - 获取查询结果集每列数据的属性（数据类型、名字、字节数），与taos_num_fileds配合使用，可用来解析`taos_fetch_row`返回的一个元组(一行)的数据。 `TAOS_FIELD` 的结构如下： + 获取查询结果集每列数据的属性（列的名称、列的数据类型、列的长度），与taos_num_fileds配合使用，可用来解析`taos_fetch_row`返回的一个元组(一行)的数据。 `TAOS_FIELD` 的结构如下： ```c typedef struct taosField { char name[65]; // 列名 uint8_t type; // 数据类型 - int16_t bytes; // 字节数 + int16_t bytes; // 长度，单位是字节 } TAOS_FIELD; ``` @@ -440,18 +456,30 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时 取消订阅。 如参数 `keepProgress` 不为0，API会保留订阅的进度信息，后续调用 `taos_subscribe` 时可以基于此进度继续；否则将删除进度信息，后续只能重新开始读取数据。 + + + ## Python Connector Python连接器的使用参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1963.html) -### 安装准备 -* 应用驱动安装请参考[安装连接器驱动步骤](https://www.taosdata.com/cn/documentation/connector#driver)。 -* 已安装python 2.7 or >= 3.4 -* 已安装pip 或 pip3 +**安装**：参见下面具体步骤 -### Python客户端安装 +**示例程序**：位于install_directory/examples/python -#### Linux +### 安装 + +Python连接器支持的系统有：Linux 64/Windows x64 + +安装前准备： + +- 已安装好TDengine应用驱动，请参考[安装连接器驱动步骤](https://www.taosdata.com/cn/documentation/connector#driver) +- 已安装python 2.7 or >= 3.4 +- 已安装pip + +### Python连接器安装 + +**Linux** 用户可以在源代码的src/connector/python（或者tar.gz的/connector/python）文件夹下找到connector安装包。用户可以通过pip命令安装： @@ -461,9 +489,10 @@ Python连接器的使用参见[视频教程](https://www.taosdata.com/blog/2020/ ​ `pip3 install src/connector/python/` -#### Windows -在已安装Windows TDengine 客户端的情况下， 将文件"C:\TDengine\driver\taos.dll" 拷贝到 "C:\windows\system32" 目录下, 然后进入Windwos cmd 命令行界面 -```cmd +**Windows** + +在已安装Windows TDengine 客户端的情况下， 将文件"C:\TDengine\driver\taos.dll" 拷贝到 "C:\Windows\system32" 目录下, 然后进入Windows *cmd* 命令行界面 +```bash cd C:\TDengine\connector\python python -m pip install . ``` @@ -471,7 +500,39 @@ python -m pip install . * 如果机器上没有pip命令，用户可将src/connector/python下的taos文件夹拷贝到应用程序的目录使用。 对于windows 客户端，安装TDengine windows 客户端后，将C:\TDengine\driver\taos.dll拷贝到C:\windows\system32目录下即可。 -### 使用 +### 示例程序 + +示例程序源码位于install_directory/examples/Python，有： +**read_example.py Python示例源程序** + +用户可以参考read_example.py这个程序来设计用户自己的写入、查询程序。 + +在安装了对应的应用驱动后，通过import taos引入taos类。主要步骤如下： + +- 通过taos.connect获取TDengineConnection对象，这个对象可以一个程序只申请一个，在多线程中共享。 + +- 通过TDengineConnection对象的 .cursor()方法获取一个新的游标对象，这个游标对象必须保证每个线程独享。 + +- 通过游标对象的execute()方法，执行写入或查询的SQL语句 + +- 如果执行的是写入语句，execute返回的是成功写入的行数信息affected rows + +- 如果执行的是查询语句，则execute执行成功后，需要通过fetchall方法去拉取结果集。 具体方法可以参考示例代码。 + + +### 安装验证 + +运行如下指令： + +```bash +cd {install_directory}/examples/python/PYTHONConnectorChecker/` +python3 PythonChecker.py -host +``` + +验证通过将打印出成功信息。 + + +### Python连接器的使用 #### 代码示例 @@ -486,7 +547,7 @@ import taos conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos") c1 = conn.cursor() ``` -* host 是TDengine 服务端所有IP, config 为客户端配置文件所在目录 +* *host* 是TDengine 服务端所有IP, *config* 为客户端配置文件所在目录 * 写入数据 @@ -599,18 +660,46 @@ conn.close() 注意：与标准连接器的一个区别是，RESTful 接口是无状态的，因此 `USE db_name` 指令没有效果，所有对表名、超级表名的引用都需要指定数据库名前缀。 -### HTTP请求格式 +### 安装 + +RESTful接口不依赖于任何TDengine的库，因此客户端不需要安装任何TDengine的库，只要客户端的开发语言支持HTTP协议即可。 + +### 验证 + +在已经安装TDengine服务器端的情况下，可以按照如下方式进行验证。 + +下面以Ubuntu环境中使用curl工具（确认已经安装）来验证RESTful接口的正常。 + +下面示例是列出所有的数据库，请把h1.taosdata.com和6041（缺省值）替换为实际运行的TDengine服务fqdn和端口号： +```html +curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'show databases;' h1.taosdata.com:6041/rest/sql +``` +返回值结果如下表示验证通过： +```json +{ + "status": "succ", + "head": ["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","status"], + "data": [ + ["log","2020-09-02 17:23:00.039",4,1,1,1,10,"30,30,30",1,3,100,4096,1,3000,2,"us","ready"], + ], + "rows": 1 +} +``` + +### RESTful连接器的使用 + +#### HTTP请求格式 ``` -http://:/rest/sql +http://:/rest/sql ``` 参数说明： -- IP: 集群中的任一台主机 -- PORT: 配置文件中httpPort配置项，缺省为6041 +- fqnd: 集群中的任一台主机FQDN或IP地址 +- port: 配置文件中httpPort配置项，缺省为6041 -例如：http://192.168.0.1:6041/rest/sql 是指向IP地址为192.168.0.1的URL. +例如：http://h1.taos.com:6041/rest/sql 是指向地址为h1.taos.com:6041的url。 HTTP请求的Header里需带有身份认证信息，TDengine支持Basic认证与自定义认证两种机制，后续版本将提供标准安全的数字签名机制来做身份验证。 @@ -662,8 +751,8 @@ curl -u username:password -d '' :/rest/sql 说明： - status: 告知操作结果是成功还是失败。 -- head: 表的定义，如果不返回结果集，则仅有一列“affected_rows”。（从 2.0.17 版本开始，建议不要依赖 head 返回值来判断数据列类型，而推荐使用 column_meta。在未来版本中，有可能会从返回值中去掉 head 这一项。） -- column_meta: 从 2.0.17 版本开始，返回值中增加这一项来说明 data 里每一列的数据类型。具体每个列会用三个值来说明，分别为：列名、列类型、类型长度。例如`["current",6,4]`表示列名为“current”；列类型为 6，也即 float 类型；类型长度为 4，也即对应 4 个字节表示的 float。如果列类型为 binary 或 nchar，则类型长度表示该列最多可以保存的内容长度，而不是本次返回值中的具体数据长度。当列类型是 nchar 的时候，其类型长度表示可以保存的 unicode 字符数量，而不是 bytes。 +- head: 表的定义，如果不返回结果集，则仅有一列“affected_rows”。（从 2.0.17.0 版本开始，建议不要依赖 head 返回值来判断数据列类型，而推荐使用 column_meta。在未来版本中，有可能会从返回值中去掉 head 这一项。） +- column_meta: 从 2.0.17.0 版本开始，返回值中增加这一项来说明 data 里每一列的数据类型。具体每个列会用三个值来说明，分别为：列名、列类型、类型长度。例如`["current",6,4]`表示列名为“current”；列类型为 6，也即 float 类型；类型长度为 4，也即对应 4 个字节表示的 float。如果列类型为 binary 或 nchar，则类型长度表示该列最多可以保存的内容长度，而不是本次返回值中的具体数据长度。当列类型是 nchar 的时候，其类型长度表示可以保存的 unicode 字符数量，而不是 bytes。 - data: 具体返回的数据，一行一行的呈现，如果不返回结果集，那么就仅有[[affected_rows]]。data 中每一行的数据列顺序，与 column_meta 中描述数据列的顺序完全一致。 - rows: 表明总共多少行数据。 @@ -684,16 +773,16 @@ column_meta 中的列类型说明： HTTP请求中需要带有授权码``，用于身份识别。授权码通常由管理员提供，可简单的通过发送`HTTP GET`请求来获取授权码，操作如下： ```bash -curl http://:6041/rest/login// +curl http://:/rest/login// ``` -其中，`ip`是TDengine数据库的IP地址，`username`为数据库用户名，`password`为数据库密码，返回值为`JSON`格式，各字段含义如下： +其中，`fqdn`是TDengine数据库的fqdn或ip地址，port是TDengine服务的端口号，`username`为数据库用户名，`password`为数据库密码，返回值为`JSON`格式，各字段含义如下： - status：请求结果的标志位 - code：返回值代码 -- desc: 授权码 +- desc：授权码 获取授权码示例： @@ -802,10 +891,10 @@ HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间 下面仅列出一些与RESTful接口有关的配置参数，其他系统参数请看配置文件里的说明。注意：配置修改后，需要重启taosd服务才能生效 - 对外提供RESTful服务的端口号，默认绑定到 6041（实际取值是 serverPort + 11，因此可以通过修改 serverPort 参数的设置来修改） -- httpMaxThreads: 启动的线程数量，默认为2（2.0.17版本开始，默认值改为CPU核数的一半向下取整） +- httpMaxThreads: 启动的线程数量，默认为2（2.0.17.0版本开始，默认值改为CPU核数的一半向下取整） - restfulRowLimit: 返回结果集（JSON格式）的最大条数，默认值为10240 - httpEnableCompress: 是否支持压缩，默认不支持，目前TDengine仅支持gzip压缩格式 -- httpDebugFlag: 日志开关，131：仅错误和报警信息，135：调试信息，143：非常详细的调试信息，默认131 +- httpDebugFlag: 日志开关，默认131。131：仅错误和报警信息，135：调试信息，143：非常详细的调试信息，默认131 ## CSharp Connector @@ -817,6 +906,12 @@ C#连接器支持的系统有：Linux 64/Windows x64/Windows x86 * .NET接口文件TDengineDrivercs.cs和参考程序示例TDengineTest.cs均位于Windows客户端install_directory/examples/C#目录下。 * 在Windows系统上，C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作，后续版本将提供ORM（Dapper）框架驱动。 +### 示例程序 + +示例程序源码位于install_directory/examples/C#，有： + +TDengineTest.cs C#示例源程序 + ### 安装验证 运行install_directory/examples/C#/C#Checker/C#Checker.exe @@ -856,32 +951,52 @@ https://www.taosdata.com/blog/2020/11/02/1901.html ### 安装准备 -* 应用驱动安装请参考[安装连接器驱动步骤](https://www.taosdata.com/cn/documentation/connector#driver)。 +Go连接器支持的系统有： -TDengine提供了GO驱动程序`taosSql`。 `taosSql`实现了GO语言的内置接口`database/sql/driver`。用户只需按如下方式引入包就可以在应用程序中访问TDengine, 详见`https://github.com/taosdata/driver-go/blob/develop/taosSql/driver_test.go`。 +| **CPU类型** | x64（64bit） | | | aarch64 | aarch32 | +| --------------- | ------------ | -------- | -------- | -------- | ---------- | +| **OS类型** | Linux | Win64 | Win32 | Linux | Linux | +| **支持与否** | **支持** | **支持** | **支持** | **支持** | **开发中** | + +安装前准备： + +- 已安装好TDengine应用驱动，参考[安装连接器驱动步骤](https://www.taosdata.com/cn/documentation/connector#driver) + +### 示例程序 使用 Go 连接器的示例代码请参考 https://github.com/taosdata/TDengine/tree/develop/tests/examples/go 以及[视频教程](https://www.taosdata.com/blog/2020/11/11/1951.html)。 -```Go +示例程序源码也位于安装目录下的 examples/go/taosdemo.go 文件中 + +**提示：建议Go版本是1.13及以上，并开启模块支持：** +```sh + go env -w GO111MODULE=on + go env -w GOPROXY=https://goproxy.io,direct +``` +在taosdemo.go所在目录下进行编译和执行： +```sh + go mod init *demo* + go build ./demo -h fqdn -p serverPort +``` + +### Go连接器的使用 + +TDengine提供了GO驱动程序包`taosSql`.`taosSql`实现了GO语言的内置接口`database/sql/driver`。用户只需按如下方式引入包就可以在应用程序中访问TDengine。 +```go import ( - "database/sql" - _ "github.com/taosdata/driver-go/taosSql" + "database/sql" + _ "github.com/taosdata/driver-go/taosSql" ) ``` -**建议使用Go版本1.13或以上，并开启模块支持：** - -```bash -go env -w GO111MODULE=on -go env -w GOPROXY=https://goproxy.io,direct -``` +**提示**：下划线与双引号之间必须有一个空格。 ### 常用API - `sql.Open(DRIVER_NAME string, dataSourceName string) *DB` - 该API用来打开DB，返回一个类型为\*DB的对象，一般情况下，DRIVER_NAME设置为字符串`taosSql`, dataSourceName设置为字符串`user:password@/tcp(host:port)/dbname`，如果客户想要用多个goroutine并发访问TDengine, 那么需要在各个goroutine中分别创建一个sql.Open对象并用之访问TDengine + 该API用来打开DB，返回一个类型为\*DB的对象，一般情况下，DRIVER_NAME设置为字符串`taosSql`，dataSourceName设置为字符串`user:password@/tcp(host:port)/dbname`，如果客户想要用多个goroutine并发访问TDengine, 那么需要在各个goroutine中分别创建一个sql.Open对象并用之访问TDengine - **注意**： 该API成功创建的时候，并没有做权限等检查，只有在真正执行Query或者Exec的时候才能真正的去创建连接，并同时检查user/password/host/port是不是合法。 另外，由于整个驱动程序大部分实现都下沉到taosSql所依赖的libtaos中。所以，sql.Open本身特别轻量。 + **注意**： 该API成功创建的时候，并没有做权限等检查，只有在真正执行Query或者Exec的时候才能真正的去创建连接，并同时检查user/password/host/port是不是合法。另外，由于整个驱动程序大部分实现都下沉到taosSql所依赖的libtaos动态库中。所以，sql.Open本身特别轻量。 - `func (db *DB) Exec(query string, args ...interface{}) (Result, error)` @@ -957,12 +1072,12 @@ npm install td2.0-connector 手动安装以下工具: - 安装Visual Studio相关：[Visual Studio Build 工具](https://visualstudio.microsoft.com/thank-you-downloading-visual-studio/?sku=BuildTools) 或者 [Visual Studio 2017 Community](https://visualstudio.microsoft.com/pl/thank-you-downloading-visual-studio/?sku=Community) -- 安装 [Python](https://www.python.org/downloads/) 2.7(`v3.x.x` 暂不支持) 并执行 `npm config set python python2.7` -- 进入`cmd`命令行界面, `npm config set msvs_version 2017` +- 安装 [Python](https://www.python.org/downloads/) 2.7(`v3.x.x` 暂不支持) 并执行 `npm config set python python2.7` +- 进入`cmd`命令行界面，`npm config set msvs_version 2017` -如果以上步骤不能成功执行, 可以参考微软的node.js用户手册[Microsoft's Node.js Guidelines for Windows](https://github.com/Microsoft/nodejs-guidelines/blob/master/windows-environment.md#compiling-native-addon-modules) +如果以上步骤不能成功执行，可以参考微软的node.js用户手册[Microsoft's Node.js Guidelines for Windows](https://github.com/Microsoft/nodejs-guidelines/blob/master/windows-environment.md#compiling-native-addon-modules) -如果在Windows 10 ARM 上使用ARM64 Node.js, 还需添加 "Visual C++ compilers and libraries for ARM64" 和 "Visual C++ ATL for ARM64". +如果在Windows 10 ARM 上使用ARM64 Node.js，还需添加 "Visual C++ compilers and libraries for ARM64" 和 "Visual C++ ATL for ARM64"。 ### 示例程序 @@ -979,7 +1094,7 @@ Node-example-raw.js 1. 新建安装验证目录，例如：`~/tdengine-test`，拷贝github上nodejsChecker.js源程序。下载地址：（https://github.com/taosdata/TDengine/tree/develop/tests/examples/nodejs/nodejsChecker.js）。 -2. 在命令中执行以下命令： +2. 在命令行中执行以下命令： ```bash npm init -y @@ -991,23 +1106,19 @@ node nodejsChecker.js host=localhost ### Node.js连接器的使用 -以下是node.js 连接器的一些基本使用方法，详细的使用方法可参考[TDengine Node.js connector](http://docs.taosdata.com/node) +以下是Node.js 连接器的一些基本使用方法，详细的使用方法可参考[TDengine Node.js connector](http://docs.taosdata.com/node)。 #### 建立连接 -使用node.js连接器时，必须先require `td2.0-connector`，然后使用 `taos.connect` 函数。`taos.connect` 函数必须提供的参数是`host`，其它参数在没有提供的情况下会使用如下的默认值。最后需要初始化`cursor` 来和TDengine服务端通信 +使用node.js连接器时，必须先`require td2.0-connector`，然后使用 `taos.connect` 函数建立到服务端的连接。例如如下代码： ```javascript const taos = require('td2.0-connector'); -var conn = taos.connect({host:"127.0.0.1", user:"root", password:"taosdata", config:"/etc/taos",port:0}) +var conn = taos.connect({host:"taosdemo.com", user:"root", password:"taosdata", config:"/etc/taos",port:6030}) var cursor = conn.cursor(); // Initializing a new cursor ``` -关闭连接可执行 - -```javascript -conn.close(); -``` +建立了一个到hostname为taosdemo.com，端口为6030（Tdengine的默认端口号）的连接。连接指定了用户名（root）和密码（taosdata）。taos.connect 函数必须提供的参数是`host`，其它参数在没有提供的情况下会使用如下的默认值。taos.connect返回了`cursor` 对象，使用cursor来执行sql语句。 #### 执行SQL和插入数据 @@ -1061,6 +1172,14 @@ promise.then(function(result) { result.pretty(); }) ``` + +#### 关闭连接 + +在完成插入、查询等操作后，要关闭连接。代码如下： +```js +conn.close(); +``` + #### 异步函数 异步查询数据库的操作和上面类似，只需要在`cursor.execute`, `TaosQuery.execute`等函数后面加上`_a`。 From f6710ee891574745b5e95f9c216eaff5eead075c Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Sun, 15 Aug 2021 22:36:37 +0800 Subject: [PATCH 71/75] [TD-4181] : update Advanced-features related doc. --- documentation20/cn/07.advanced-features/docs.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/documentation20/cn/07.advanced-features/docs.md b/documentation20/cn/07.advanced-features/docs.md index 1b4ccb4814..3661d68427 100644 --- a/documentation20/cn/07.advanced-features/docs.md +++ b/documentation20/cn/07.advanced-features/docs.md @@ -138,7 +138,7 @@ select * from meters where ts > now - 1d and current > 10; 订阅的`topic`实际上是它的名字，因为订阅功能是在客户端API中实现的，所以没必要保证它全局唯一，但需要它在一台客户端机器上唯一。 -如果名`topic`的订阅不存在，参数`restart`没有意义；但如果用户程序创建这个订阅后退出，当它再次启动并重新使用这个`topic`时，`restart`就会被用于决定是从头开始读取数据，还是接续上次的位置进行读取。本例中，如果`restart`是 **true**（非零值），用户程序肯定会读到所有数据。但如果这个订阅之前就存在了，并且已经读取了一部分数据，且`restart`是 **false**（**0**），用户程序就不会读到之前已经读取的数据了。 +如果名为`topic`的订阅不存在，参数`restart`没有意义；但如果用户程序创建这个订阅后退出，当它再次启动并重新使用这个`topic`时，`restart`就会被用于决定是从头开始读取数据，还是接续上次的位置进行读取。本例中，如果`restart`是 **true**（非零值），用户程序肯定会读到所有数据。但如果这个订阅之前就存在了，并且已经读取了一部分数据，且`restart`是 **false**（**0**），用户程序就不会读到之前已经读取的数据了。 `taos_subscribe`的最后一个参数是以毫秒为单位的轮询周期。在同步模式下，如果前后两次调用`taos_consume`的时间间隔小于此时间，`taos_consume`会阻塞，直到间隔超过此时间。异步模式下，这个时间是两次调用回调函数的最小时间间隔。 @@ -179,7 +179,8 @@ void print_result(TAOS_RES* res, int blockFetch) { 　　} else { 　　　　while ((row = taos_fetch_row(res))) { 　　　　　　char temp[256]; -　　　　　　taos_print_row(temp, row, fields, num_fields);puts(temp); +　　　　　　taos_print_row(temp, row, fields, num_fields); +　　　　　　puts(temp); 　　　　　　nRows++; 　　　　} 　　} @@ -211,14 +212,14 @@ taos_unsubscribe(tsub, keep); 则可以在示例代码所在目录执行以下命令来编译并启动示例程序： -```shell +```bash $ make $ ./subscribe -sql='select * from meters where current > 10;' ``` 示例程序启动后，打开另一个终端窗口，启动 TDengine 的 shell 向 **D1001** 插入一条电流为 12A 的数据： -```shell +```sql $ taos > use test; > insert into D1001 values(now, 12, 220, 1); @@ -313,7 +314,7 @@ public class SubscribeDemo { 运行示例程序，首先，它会消费符合查询条件的所有历史数据： -```shell +```bash # java -jar subscribe.jar ts: 1597464000000 current: 12.0 voltage: 220 phase: 1 location: Beijing.Chaoyang groupid : 2 @@ -333,16 +334,16 @@ taos> insert into d1001 values("2020-08-15 12:40:00.000", 12.4, 220, 1); 因为这条数据的电流大于10A，示例程序会将其消费： -```shell +``` ts: 1597466400000 current: 12.4 voltage: 220 phase: 1 location: Beijing.Chaoyang groupid: 2 ``` ## 缓存（Cache） -TDengine采用时间驱动缓存管理策略（First-In-First-Out，FIFO），又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式（Least-Recent-Use，LRU），直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候，将最早的数据批量写入磁盘。一般意义上来说，对于物联网数据的使用，用户最为关心最近产生的数据，即当前状态。TDengine充分利用了这一特性，将最近到达的（当前状态）数据保存在缓存中。 +TDengine采用时间驱动缓存管理策略（First-In-First-Out，FIFO），又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式（Least-Recent-Used，LRU），直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候，将最早的数据批量写入磁盘。一般意义上来说，对于物联网数据的使用，用户最为关心最近产生的数据，即当前状态。TDengine充分利用了这一特性，将最近到达的（当前状态）数据保存在缓存中。 -TDengine通过查询函数向用户提供毫秒级的数据获取能力。直接将最近到达的数据保存在缓存中，可以更加快速地响应用户针对最近一条或一批数据的查询分析，整体上提供更快的数据库查询响应能力。从这个意义上来说，可通过设置合适的配置参数将TDengine作为数据缓存来使用，而不需要再部署额外的缓存系统，可有效地简化系统架构，降低运维的成本。需要注意的是，TDengine重启以后系统的缓存将被清空，之前缓存的数据均会被批量写入磁盘，缓存的数据将不会像专门的Key-value缓存系统再将之前缓存的数据重新加载到缓存中。 +TDengine通过查询函数向用户提供毫秒级的数据获取能力。直接将最近到达的数据保存在缓存中，可以更加快速地响应用户针对最近一条或一批数据的查询分析，整体上提供更快的数据库查询响应能力。从这个意义上来说，可通过设置合适的配置参数将TDengine作为数据缓存来使用，而不需要再部署额外的缓存系统，可有效地简化系统架构，降低运维的成本。需要注意的是，TDengine重启以后系统的缓存将被清空，之前缓存的数据均会被批量写入磁盘，缓存的数据将不会像专门的key-value缓存系统再将之前缓存的数据重新加载到缓存中。 TDengine分配固定大小的内存空间作为缓存空间，缓存空间可根据应用的需求和硬件资源配置。通过适当的设置缓存空间，TDengine可以提供极高性能的写入和查询的支持。TDengine中每个虚拟节点（virtual node）创建时分配独立的缓存池。每个虚拟节点管理自己的缓存池，不同虚拟节点间不共享缓存池。每个虚拟节点内部所属的全部表共享该虚拟节点的缓存池。 From b32fca9dec8d6c2d3cd91998c72033be31ad46a5 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Sun, 15 Aug 2021 22:45:21 +0800 Subject: [PATCH 72/75] [TD-4181] : update Queries related doc. --- documentation20/cn/06.queries/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/06.queries/docs.md b/documentation20/cn/06.queries/docs.md index 5557134aac..294a9721e1 100644 --- a/documentation20/cn/06.queries/docs.md +++ b/documentation20/cn/06.queries/docs.md @@ -6,7 +6,7 @@ TDengine 采用 SQL 作为查询语言。应用程序可以通过 C/C++, Java, Go, Python 连接器发送 SQL 语句，用户可以通过 TDengine 提供的命令行（Command Line Interface, CLI）工具 TAOS Shell 手动执行 SQL 即席查询（Ad-Hoc Query）。TDengine 支持如下查询功能： - 单列、多列数据查询 -- 标签和数值的多种过滤条件：>, <, =, <>, like 等 +- 标签和数值的多种过滤条件：>, <, =, <>, like 等 - 聚合结果的分组（Group by）、排序（Order by）、约束输出（Limit/Offset） - 数值列及聚合结果的四则运算 - 时间戳对齐的连接查询（Join Query: 隐式连接）操作 From da8ebbef3e5e7b29ad17c224d5cc65723c912373 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 16 Aug 2021 09:37:13 +0800 Subject: [PATCH 73/75] [ci skip] update daily performance script --- tests/pytest/tools/taosdemoPerformance.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index 4a5abd49d8..1d28a2708f 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -145,26 +145,26 @@ class taosdemoPerformace: binPath = buildPath + "/build/bin/" os.system( - "%staosdemo -f %s > taosdemoperf.txt 2>&1" % + "%staosdemo -f %s > /dev/null 2>&1" % (binPath, self.generateJson())) self.createTableTime = self.getCMDOutput( - "grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'") + "grep 'Spent' insert_res.txt | awk 'NR==1{print $2}'") self.insertRecordsTime = self.getCMDOutput( - "grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'") + "grep 'Spent' insert_res.txt | awk 'NR==2{print $2}'") self.recordsPerSecond = self.getCMDOutput( - "grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'") + "grep 'Spent' insert_res.txt | awk 'NR==2{print $16}'") self.commitID = self.getCMDOutput("git rev-parse --short HEAD") delay = self.getCMDOutput( - "grep 'delay' taosdemoperf.txt | awk '{print $4}'") + "grep 'delay' insert_res.txt | awk '{print $4}'") self.avgDelay = delay[:-4] delay = self.getCMDOutput( - "grep 'delay' taosdemoperf.txt | awk '{print $6}'") + "grep 'delay' insert_res.txt | awk '{print $6}'") self.maxDelay = delay[:-4] delay = self.getCMDOutput( - "grep 'delay' taosdemoperf.txt | awk '{print $8}'") + "grep 'delay' insert_res.txt | awk '{print $8}'") self.minDelay = delay[:-3] - os.system("[ -f taosdemoperf.txt ] && rm taosdemoperf.txt") + os.system("[ -f insert_res.txt ] && rm insert_res.txt") def createTablesAndStoreData(self): cursor = self.conn2.cursor() @@ -185,7 +185,7 @@ class taosdemoPerformace: cursor.close() cursor1 = self.conn.cursor() - # cursor1.execute("drop database if exists %s" % self.insertDB) + cursor1.execute("drop database if exists %s" % self.insertDB) cursor1.close() if __name__ == '__main__': From d3cf95e72b84d9cd90efe8c05d89fc837c3be703 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Mon, 16 Aug 2021 10:50:39 +0800 Subject: [PATCH 74/75] [TD-5532] : fix Grafana plugin json file location. --- documentation20/cn/09.connections/docs.md | 2 +- documentation20/en/09.connections/docs.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation20/cn/09.connections/docs.md b/documentation20/cn/09.connections/docs.md index 7c70724823..03ab201815 100644 --- a/documentation20/cn/09.connections/docs.md +++ b/documentation20/cn/09.connections/docs.md @@ -64,7 +64,7 @@ sudo cp -rf /usr/local/taos/connector/grafanaplugin /var/lib/grafana/plugins/tde #### 导入 Dashboard -在 Grafana 插件目录 /usr/local/taos/connector/grafana/tdengine/dashboard/ 下提供了一个 `tdengine-grafana.json` 可导入的 dashboard。 +在 Grafana 插件目录 /usr/local/taos/connector/grafanaplugin/dashboard 下提供了一个 `tdengine-grafana.json` 可导入的 dashboard。 点击左侧 `Import` 按钮，并上传 `tdengine-grafana.json` 文件： diff --git a/documentation20/en/09.connections/docs.md b/documentation20/en/09.connections/docs.md index e759da3167..b693d228cf 100644 --- a/documentation20/en/09.connections/docs.md +++ b/documentation20/en/09.connections/docs.md @@ -6,7 +6,7 @@ TDengine can quickly integrate with [Grafana](https://www.grafana.com/), an open ### Install Grafana -TDengine currently supports Grafana 5.2.4 and above. You can download and install the package from Grafana website according to the current operating system. The download address is as follows: +TDengine currently supports Grafana 6.2 and above. You can download and install the package from Grafana website according to the current operating system. The download address is as follows: https://grafana.com/grafana/download. @@ -64,7 +64,7 @@ According to the default prompt, query the average system memory usage at the sp #### Import Dashboard -A `tdengine-grafana.json` importable dashboard is provided under the Grafana plug-in directory/usr/local/taos/connector/grafana/tdengine/dashboard/. +A `tdengine-grafana.json` importable dashboard is provided under the Grafana plug-in directory `/usr/local/taos/connector/grafanaplugin/dashboard`. Click the `Import` button on the left panel and upload the `tdengine-grafana.json` file: From e11f30466a456a1b8b526248eb7995155b6ce0d3 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 16 Aug 2021 11:38:16 +0800 Subject: [PATCH 75/75] [ci skip] --- src/wal/src/walWrite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index e991bf02aa..cfadafebdd 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -540,7 +540,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch pWal->version = pHead->version; - // wInfo("writeFp: %ld", offset); + //wInfo("writeFp: %ld", offset); if (0 != walSMemRowCheck(pHead)) { wError("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, fileId, pHead->version, pWal->version, pHead->len, offset);
对比项JDBC-JNIJDBC-RESTful