From f007bdf8f69412c8cd3119f20550d80b50da4e5d Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 26 Jun 2020 15:40:15 +0800 Subject: [PATCH 01/64] fix some coverity issues --- src/client/src/tscLocalMerge.c | 5 ++++- src/client/src/tscParseInsert.c | 5 ++++- src/client/src/tscSubquery.c | 6 ++++-- src/cq/test/cqtest.c | 2 +- src/query/src/qExecutor.c | 2 +- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index a731fbae8e..db85b7a5e5 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -315,7 +315,10 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->finalRowSize = tscGetResRowLength(pQueryInfo->exprList); pReducer->resColModel = finalmodel; - pReducer->resColModel->capacity = pReducer->nResultBufSize / pReducer->finalRowSize; + pReducer->resColModel->capacity = pReducer->nResultBufSize; + if (pReducer->finalRowSize > 0) { + pReducer->resColModel->capacity /= pReducer->finalRowSize; + } assert(pReducer->finalRowSize <= pReducer->rowSize); pReducer->pFinalRes = calloc(1, pReducer->rowSize * pReducer->resColModel->capacity); diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 1db4108d22..c237dbbe1d 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -802,7 +802,10 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } STableMetaInfo *pSTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); - tscSetTableFullName(pSTableMeterMetaInfo, &sToken, pSql); + code = tscSetTableFullName(pSTableMeterMetaInfo, &sToken, pSql); + if (code != TSDB_CODE_SUCCESS) { + return code; + } tstrncpy(pTag->name, pSTableMeterMetaInfo->name, sizeof(pTag->name)); code = tscGetTableMeta(pSql, pSTableMeterMetaInfo); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index e6daab99c6..32b8129d02 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -936,8 +936,10 @@ static SJoinSupporter* tscUpdateSubqueryStatus(SSqlObj* pSql, int32_t numOfFetch } } - pState->numOfTotal = pSql->numOfSubs; - pState->numOfRemain = numOfFetch; + if (pState != NULL) { + pState->numOfTotal = pSql->numOfSubs; + pState->numOfRemain = numOfFetch; + } return pSupporter; } diff --git a/src/cq/test/cqtest.c b/src/cq/test/cqtest.c index d06f14d53f..1daee644a7 100644 --- a/src/cq/test/cqtest.c +++ b/src/cq/test/cqtest.c @@ -76,7 +76,7 @@ int main(int argc, char *argv[]) { tdFreeSchema(pSchema); while (1) { - char c = getchar(); + char c = (char)getchar(); switch(c) { case 's': diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index ed529db92e..76bdaf7c36 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5901,7 +5901,7 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo)) != TSDB_CODE_SUCCESS) { - return code; + goto _over; } if (pQueryMsg->numOfTables <= 0) { From e7f5185b85b33b446a20dfb342128547ff34d2bb Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 27 Jun 2020 13:18:06 +0000 Subject: [PATCH 02/64] memory leak for redirection case --- src/dnode/src/dnodeMPeer.c | 1 + src/dnode/src/dnodeMRead.c | 1 + src/dnode/src/dnodeMWrite.c | 1 + src/dnode/src/dnodeShell.c | 6 +++--- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c index c37b2bed79..92fb1bd417 100644 --- a/src/dnode/src/dnodeMPeer.c +++ b/src/dnode/src/dnodeMPeer.c @@ -113,6 +113,7 @@ void dnodeFreeMnodePqueue() { void dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg) { if (!mnodeIsRunning() || tsMPeerQueue == NULL) { dnodeSendRedirectMsg(pMsg, false); + rpcFreeCont(pMsg->pCont); return; } diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index cb02ffbb1d..430f0daa59 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -116,6 +116,7 @@ void dnodeFreeMnodeRqueue() { void dnodeDispatchToMnodeReadQueue(SRpcMsg *pMsg) { if (!mnodeIsRunning() || tsMReadQueue == NULL) { dnodeSendRedirectMsg(pMsg, true); + rpcFreeCont(pMsg->pCont); return; } diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index 47645ea5ea..35657a6e45 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -115,6 +115,7 @@ void dnodeFreeMnodeWqueue() { void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg) { if (!mnodeIsRunning() || tsMWriteQueue == NULL) { dnodeSendRedirectMsg(pMsg, true); + rpcFreeCont(pMsg->pCont); return; } diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index 4252e63f8d..1e0f1a6415 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -38,9 +38,9 @@ static int32_t tsDnodeQueryReqNum = 0; static int32_t tsDnodeSubmitReqNum = 0; int32_t dnodeInitShell() { - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeDispatchToVnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY] = dnodeDispatchToVnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH] = dnodeDispatchToVnodeReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeDispatchToVnodeWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY] = dnodeDispatchToVnodeReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH] = dnodeDispatchToVnodeReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVnodeWriteQueue; // the following message shall be treated as mnode write From eb54250708b6457d7fc7f6ba4b739ba14687571d Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 28 Jun 2020 05:43:34 +0000 Subject: [PATCH 03/64] memory leak for redirection case --- src/dnode/src/dnodeVRead.c | 1 + src/dnode/src/dnodeVWrite.c | 1 + 2 files changed, 2 insertions(+) diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index cd18ae6dda..3027a94411 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -131,6 +131,7 @@ void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg) { .msgType = 0 }; rpcSendResponse(&rpcRsp); + rpcFreeCont(pMsg->pCont); } } diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index e61364355d..e2cc2d1cd3 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -119,6 +119,7 @@ void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg) { .msgType = 0 }; rpcSendResponse(&rpcRsp); + rpcFreeCont(pMsg->pCont); } } From 912137cf4686c67d1bb0c100619d2d44eeca56c4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 28 Jun 2020 16:26:08 +0800 Subject: [PATCH 04/64] [td-225] add error check --- src/vnode/src/vnodeRead.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 91c6d28e22..013f8c85ac 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -110,7 +110,8 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { pRet->rsp = pRsp; // current connect is broken - if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, pQInfo, pVnode->vgId) != TSDB_CODE_SUCCESS) { + if ((code == TSDB_CODE_SUCCESS) && + (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, pQInfo, pVnode->vgId) != TSDB_CODE_SUCCESS)) { vError("vgId:%d, QInfo:%p, dnode query discarded since link is broken, %p", pVnode->vgId, pQInfo, pReadMsg->rpcMsg.handle); pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; From d748013039ea662d3cbddeb424a98f633064fd99 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 28 Jun 2020 16:58:25 +0800 Subject: [PATCH 05/64] [td-225] handle error code. --- src/tsdb/src/tsdbRWHelper.c | 4 +-- src/tsdb/src/tsdbRead.c | 60 ++++++++++++++++++++++++------------- src/vnode/src/vnodeRead.c | 9 ++++-- 3 files changed, 48 insertions(+), 25 deletions(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 2ab0a7d0a2..b5f89d50c8 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -120,7 +120,7 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { tsdbError("vgId:%d failed to sendfile %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), TSDB_FILE_HEAD_SIZE, pHelper->files.headF.fname, pHelper->files.nHeadF.fname, strerror(errno)); - errno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -143,7 +143,7 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { return tsdbLoadCompIdx(pHelper, NULL); _err: - return -1; + return terrno; } int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 7135c050d5..6ae9ea18db 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -148,7 +148,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab pQueryHandle->type = TSDB_QUERY_TYPE_ALL; pQueryHandle->cur.fid = -1; pQueryHandle->cur.win = TSWINDOW_INITIALIZER; - pQueryHandle->checkFiles = true;//ASCENDING_TRAVERSE(pQueryHandle->order); + pQueryHandle->checkFiles = true; pQueryHandle->activeIndex = 0; // current active table index pQueryHandle->qinfo = qinfo; pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock; @@ -475,11 +475,15 @@ static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSK } static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks, int32_t type) { - // todo check open file failed SFileGroup* fileGroup = pQueryHandle->pFileGroup; - assert(fileGroup->files[TSDB_FILE_TYPE_HEAD].fname > 0); - tsdbSetAndOpenHelperFile(&pQueryHandle->rhelper, fileGroup); + + int32_t code = tsdbSetAndOpenHelperFile(&pQueryHandle->rhelper, fileGroup); + + //open file failed, return error code to client + if (code != TSDB_CODE_SUCCESS) { + return code; + } // load all the comp offset value for all tables in this file *numOfBlocks = 0; @@ -593,8 +597,7 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo if (pCheckInfo->pDataCols == NULL) { STsdbMeta* pMeta = tsdbGetMeta(pRepo); // TODO - pCheckInfo->pDataCols = - tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock); + pCheckInfo->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock); } tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(pCheckInfo->pTableObj)); @@ -1359,16 +1362,18 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO } // todo opt for only one table case -static bool getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle) { +static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* exists) { pQueryHandle->numOfBlocks = 0; SQueryFilePos* cur = &pQueryHandle->cur; - + + int32_t code = TSDB_CODE_SUCCESS; + int32_t numOfBlocks = 0; int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { int32_t type = ASCENDING_TRAVERSE(pQueryHandle->order)? QUERY_RANGE_GREATER_EQUAL:QUERY_RANGE_LESS_EQUAL; - if (getFileCompInfo(pQueryHandle, &numOfBlocks, type) != TSDB_CODE_SUCCESS) { + if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks, type)) != TSDB_CODE_SUCCESS) { break; } @@ -1393,20 +1398,25 @@ static bool getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle) { // no data in file anymore if (pQueryHandle->numOfBlocks <= 0) { - assert(pQueryHandle->pFileGroup == NULL); + if (code == TSDB_CODE_SUCCESS) { + assert(pQueryHandle->pFileGroup == NULL); + } + cur->fid = -1; // denote that there are no data in file anymore - - return false; + *exists = false; + return code; } cur->slot = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; cur->fid = pQueryHandle->pFileGroup->fileId; STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; - return loadFileDataBlock(pQueryHandle, pBlockInfo->compBlock, pBlockInfo->pTableCheckInfo); + *exists = loadFileDataBlock(pQueryHandle, pBlockInfo->compBlock, pBlockInfo->pTableCheckInfo); + + return TSDB_CODE_SUCCESS; } -static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { +static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) { STsdbFileH* pFileHandle = tsdbGetFile(pQueryHandle->pTsdb); SQueryFilePos* cur = &pQueryHandle->cur; @@ -1419,7 +1429,7 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order); tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); - return getDataBlocksInFilesImpl(pQueryHandle); + return getDataBlocksInFilesImpl(pQueryHandle, exists); } else { // check if current file block is all consumed STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; @@ -1430,7 +1440,7 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) { // all data blocks in current file has been checked already, try next file if exists - return getDataBlocksInFilesImpl(pQueryHandle); + return getDataBlocksInFilesImpl(pQueryHandle, exists); } else { // next block of the same file int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; @@ -1440,11 +1450,15 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { cur->blockCompleted = false; STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; - return loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo); + *exists = loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo); + + return TSDB_CODE_SUCCESS; } } else { handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->compBlock, pCheckInfo); - return pQueryHandle->realNumOfRows > 0; + *exists = pQueryHandle->realNumOfRows > 0; + + return TSDB_CODE_SUCCESS; } } } @@ -1576,8 +1590,14 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { } if (pQueryHandle->checkFiles) { - if (getDataBlocksInFiles(pQueryHandle)) { - return true; + bool exists = true; + int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + + if (exists) { + return exists; } pQueryHandle->activeIndex = 0; diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 91c6d28e22..1f009c9e0d 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -110,11 +110,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { pRet->rsp = pRsp; // current connect is broken - if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, pQInfo, pVnode->vgId) != TSDB_CODE_SUCCESS) { - vError("vgId:%d, QInfo:%p, dnode query discarded since link is broken, %p", pVnode->vgId, pQInfo, pReadMsg->rpcMsg.handle); + if ((code == TSDB_CODE_SUCCESS) && + (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, pQInfo, pVnode->vgId) != TSDB_CODE_SUCCESS)) { + + vError("vgId:%d, QInfo:%p, dnode query discarded since link is broken, %p", pVnode->vgId, pQInfo, + pReadMsg->rpcMsg.handle); pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - //NOTE: there two refcount, needs to kill twice, todo refactor + // NOTE: there two refcount, needs to kill twice, todo refactor qKillQuery(pQInfo, vnodeRelease, pVnode); qKillQuery(pQInfo, vnodeRelease, pVnode); From 1d06bc7abd73c3b90c9b0fede25abb30e04026c7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 28 Jun 2020 17:11:57 +0800 Subject: [PATCH 06/64] add load column data interface --- src/common/src/tdataformat.c | 26 ++-- src/tsdb/inc/tsdbMain.h | 4 +- src/tsdb/src/tsdbRWHelper.c | 225 +++++++++++++++++++++++------------ src/tsdb/src/tsdbRead.c | 2 +- 4 files changed, 169 insertions(+), 88 deletions(-) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index cd1f4b1ca4..5610f0f75e 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -371,9 +371,11 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { if (keepData) { pRet->cols[i].len = pDataCols->cols[i].len; - memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len); - if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) { - memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, sizeof(VarDataOffsetT) * pDataCols->maxPoints); + if (pDataCols->cols[i].len > 0) { + memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len); + if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) { + memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, sizeof(VarDataOffsetT) * pDataCols->maxPoints); + } } } } @@ -443,8 +445,10 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) { if (dataColsKeyLast(target) < dataColsKeyFirst(source)) { // No overlap for (int i = 0; i < rowsToMerge; i++) { for (int j = 0; j < source->numOfCols; j++) { - dataColAppendVal(target->cols + j, tdGetColDataOfRow(source->cols + j, i), target->numOfRows, - target->maxPoints); + if (source->cols[j].len > 0) { + dataColAppendVal(target->cols + j, tdGetColDataOfRow(source->cols + j, i), target->numOfRows, + target->maxPoints); + } } target->numOfRows++; } @@ -479,8 +483,10 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi if (key1 <= key2) { for (int i = 0; i < src1->numOfCols; i++) { ASSERT(target->cols[i].type == src1->cols[i].type); - dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows, - target->maxPoints); + if (src1->cols[i].len > 0) { + dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows, + target->maxPoints); + } } target->numOfRows++; @@ -489,8 +495,10 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi } else { for (int i = 0; i < src2->numOfCols; i++) { ASSERT(target->cols[i].type == src2->cols[i].type); - dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows, - target->maxPoints); + if (src2->cols[i].len > 0) { + dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows, + target->maxPoints); + } } target->numOfRows++; diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 7365d7b151..29567d09b8 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -381,8 +381,8 @@ int tsdbLoadCompIdx(SRWHelper* pHelper, void* target); int tsdbLoadCompInfo(SRWHelper* pHelper, void* target); int tsdbLoadCompData(SRWHelper* phelper, SCompBlock* pcompblock, void* target); void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols); -int tsdbLoadBlockDataCols(SRWHelper* pHelper, SDataCols* pDataCols, int blkIdx, int16_t* colIds, int numOfColIds); -int tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock, SDataCols* target); +int tsdbLoadBlockDataCols(SRWHelper* pHelper, SCompBlock* pCompBlock, int16_t* colIds, int numOfColIds); +int tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock); // ------------------ tsdbMain.c #define REPO_ID(r) (r)->config.tsdbId diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 2ab0a7d0a2..280e64207a 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -20,6 +20,8 @@ #include "tscompression.h" #include "tsdbMain.h" +#define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM)) + static bool tsdbShouldCreateNewLast(SRWHelper *pHelper); static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, SCompBlock *pCompBlock, bool isLast, bool isSuperBlock); @@ -42,17 +44,16 @@ static void tsdbResetHelperBlockImpl(SRWHelper *pHelper); static void tsdbResetHelperBlock(SRWHelper *pHelper); static int tsdbInitHelperBlock(SRWHelper *pHelper); static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type); -static int comparColIdCompCol(const void *arg1, const void *arg2); -static int comparColIdDataCol(const void *arg1, const void *arg2); -static int tsdbLoadSingleColumnData(int fd, SCompBlock *pCompBlock, SCompCol *pCompCol, void *buf); -static int tsdbLoadSingleBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, int16_t *colIds, int numOfColIds, - SDataCols *pDataCols); static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfRows, int maxPoints, char *buffer, int bufferSize); +static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols, int16_t *colIds, + int numOfColIds); static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols); static int tsdbEncodeSCompIdx(void **buf, SCompIdx *pIdx); static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx); static void tsdbDestroyHelperBlock(SRWHelper *pHelper); +static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBlock, SCompCol *pCompCol, + SDataCol *pDataCol); // ---------------------- INTERNAL FUNCTIONS ---------------------- int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { @@ -315,7 +316,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { ASSERT(pCompBlock->last); if (pCompBlock->numOfSubBlocks > 1) { - if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, pIdx->numOfBlocks - 1), NULL) < 0) return -1; + if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, pIdx->numOfBlocks - 1)) < 0) return -1; ASSERT(pHelper->pDataCols[0]->numOfRows > 0 && pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock); if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0], pHelper->pDataCols[0]->numOfRows, &compBlock, true, true) < 0) @@ -510,14 +511,34 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) { ASSERT(pCompBlock->numOfSubBlocks <= 1); - int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd; + SFile *pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF); - if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) return -1; + if (lseek(pFile->fd, pCompBlock->offset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } - size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM); + size_t tsize = TSDB_GET_COMPCOL_LEN(pCompBlock->numOfCols); pHelper->pCompData = trealloc((void *)pHelper->pCompData, tsize); - if (pHelper->pCompData == NULL) return -1; - if (tread(fd, (void *)pHelper->pCompData, tsize) < tsize) return -1; + if (pHelper->pCompData == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + if (tread(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) { + tsdbError("vgId:%d failed to read %zu bytes from file %s since %s", REPO_ID(pHelper->pRepo), tsize, pFile->fname, + strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (!taosCheckChecksumWhole((uint8_t *)pHelper->pCompData, tsize)) { + tsdbError("vgId:%d file %s is broken, offset %" PRId64 " size %zu", REPO_ID(pHelper->pRepo), pFile->fname, + (int64_t)pCompBlock->offset, tsize); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } ASSERT(pCompBlock->numOfCols == pHelper->pCompData->numOfCols); @@ -554,30 +575,31 @@ void tsdbGetDataStatis(SRWHelper *pHelper, SDataStatis *pStatis, int numOfCols) } } -int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds) { - SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; - +int tsdbLoadBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, int16_t *colIds, int numOfColIds) { ASSERT(pCompBlock->numOfSubBlocks >= 1); // Must be super block - int numOfSubBlocks = pCompBlock->numOfSubBlocks; - SCompBlock *pStartBlock = - (numOfSubBlocks == 1) ? pCompBlock : (SCompBlock *)((char *)pHelper->pCompInfo->blocks + pCompBlock->offset); + int numOfSubBlocks = pCompBlock->numOfSubBlocks; + if (numOfSubBlocks > 1) pCompBlock = (SCompBlock *)POINTER_SHIFT(pHelper->pCompInfo, pCompBlock->offset); - if (tsdbLoadSingleBlockDataCols(pHelper, pStartBlock, colIds, numOfColIds, pDataCols) < 0) return -1; + tdResetDataCols(pHelper->pDataCols[0]); + if (tsdbLoadBlockDataColsImpl(pHelper, pCompBlock, pHelper->pDataCols[0], colIds, numOfColIds) < 0) goto _err; for (int i = 1; i < numOfSubBlocks; i++) { - pStartBlock++; - if (tsdbLoadSingleBlockDataCols(pHelper, pStartBlock, colIds, numOfColIds, pHelper->pDataCols[1]) < 0) return -1; - tdMergeDataCols(pDataCols, pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows); + tdResetDataCols(pHelper->pDataCols[1]); + pCompBlock++; + if (tsdbLoadBlockDataColsImpl(pHelper, pCompBlock, pHelper->pDataCols[1], colIds, numOfColIds) < 0) goto _err; + if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows) < 0) goto _err; } return 0; + +_err: + return -1; } -int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *target) { - // SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; +int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock) { int numOfSubBlock = pCompBlock->numOfSubBlocks; - if (numOfSubBlock > 1) pCompBlock = (SCompBlock *)((char *)pHelper->pCompInfo + pCompBlock->offset); + if (numOfSubBlock > 1) pCompBlock = (SCompBlock *)POINTER_SHIFT(pHelper->pCompInfo, pCompBlock->offset); tdResetDataCols(pHelper->pDataCols[0]); if (tsdbLoadBlockDataImpl(pHelper, pCompBlock, pHelper->pDataCols[0]) < 0) goto _err; @@ -588,8 +610,6 @@ int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *tar if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows) < 0) goto _err; } - // if (target) TODO - return 0; _err: @@ -648,7 +668,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa // Compress the data if neccessary int tcol = 0; int32_t toffset = 0; - int32_t tsize = sizeof(SCompData) + sizeof(SCompCol) * nColsNotAllNull + sizeof(TSCKSUM); + int32_t tsize = TSDB_GET_COMPCOL_LEN(nColsNotAllNull); int32_t lsize = tsize; for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { if (tcol >= nColsNotAllNull) break; @@ -770,7 +790,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err; } else { // Load - if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx), NULL) < 0) goto _err; + if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx)) < 0) goto _err; ASSERT(pHelper->pDataCols[0]->numOfRows <= blockAtIdx(pHelper, blkIdx)->numOfRows); // Merge if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsWritten) < 0) goto _err; @@ -826,7 +846,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err; } else { // Load-Merge-Write // Load - if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx), NULL) < 0) goto _err; + if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx)) < 0) goto _err; if (blockAtIdx(pHelper, blkIdx)->last) pHelper->hasOldLastBlock = false; rowsWritten = rows3; @@ -1183,52 +1203,13 @@ _err: return -1; } -static int comparColIdCompCol(const void *arg1, const void *arg2) { - return (*(int16_t *)arg1) - ((SCompCol *)arg2)->colId; -} - -static int comparColIdDataCol(const void *arg1, const void *arg2) { - return (*(int16_t *)arg1) - ((SDataCol *)arg2)->colId; -} - -static int tsdbLoadSingleColumnData(int fd, SCompBlock *pCompBlock, SCompCol *pCompCol, void *buf) { - size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols; - if (lseek(fd, pCompBlock->offset + tsize + pCompCol->offset, SEEK_SET) < 0) return -1; - if (tread(fd, buf, pCompCol->len) < pCompCol->len) return -1; - - return 0; -} - -static int tsdbLoadSingleBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, int16_t *colIds, int numOfColIds, - SDataCols *pDataCols) { - if (tsdbLoadCompData(pHelper, pCompBlock, NULL) < 0) return -1; - int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd; - - void *ptr = NULL; - for (int i = 0; i < numOfColIds; i++) { - int16_t colId = colIds[i]; - - ptr = bsearch((void *)&colId, (void *)pHelper->pCompData->cols, pHelper->pCompData->numOfCols, sizeof(SCompCol), - comparColIdCompCol); - if (ptr == NULL) continue; - SCompCol *pCompCol = (SCompCol *)ptr; - - ptr = - bsearch((void *)&colId, (void *)(pDataCols->cols), pDataCols->numOfCols, sizeof(SDataCol), comparColIdDataCol); - ASSERT(ptr != NULL); - SDataCol *pDataCol = (SDataCol *)ptr; - - pDataCol->len = pCompCol->len; - if (tsdbLoadSingleColumnData(fd, pCompBlock, pCompCol, pDataCol->pData) < 0) return -1; - } - - return 0; -} - static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfRows, int maxPoints, char *buffer, int bufferSize) { // Verify by checksum - if (!taosCheckChecksumWhole((uint8_t *)content, len)) return -1; + if (!taosCheckChecksumWhole((uint8_t *)content, len)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } // Decode the data if (comp) { @@ -1249,10 +1230,97 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32 return 0; } -static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) { +static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBlock, SCompCol *pCompCol, + SDataCol *pDataCol) { + ASSERT(pDataCol->colId == pCompCol->colId); + int tsize = pDataCol->bytes * pCompBlock->numOfRows + COMP_OVERFLOW_BYTES; + pHelper->pBuffer = trealloc(pHelper->pBuffer, pCompCol->len); + if (pHelper->pBuffer == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + pHelper->compBuffer = trealloc(pHelper->compBuffer, tsize); + if (pHelper->compBuffer == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + if (lseek(pFile->fd, pCompCol->offset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (tread(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) { + tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompCol->len, pFile->fname, + strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (tsdbCheckAndDecodeColumnData(pDataCol, pHelper->pBuffer, pCompCol->len, pCompBlock->algorithm, + pCompBlock->numOfRows, pHelper->pRepo->config.maxRowsPerFileBlock, + pHelper->compBuffer, tsizeof(pHelper->compBuffer)) < 0) { + tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pHelper->pRepo), pFile->fname, pCompCol->colId, + (int64_t)pCompCol->offset); + return -1; + } + + return 0; +} + +static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols, int16_t *colIds, int numOfColIds) { ASSERT(pCompBlock->numOfSubBlocks <= 1); - SCompData *pCompData = (SCompData *)pHelper->pBuffer; + SFile * pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF); + + if (tsdbLoadCompData(pHelper, pCompBlock, NULL) < 0) goto _err; + + int dcol = 0; + int ccol = 0; + for (int i = 0; i < numOfColIds; i++) { + int16_t colId = colIds[i]; + SDataCol *pDataCol = NULL; + SCompCol *pCompCol = NULL; + + while (true) { + ASSERT(dcol < pDataCols->numOfCols); + pDataCol = &pDataCols->cols[dcol]; + ASSERT(pDataCol->colId <= colId); + if (pDataCol->colId == colId) break; + dcol++; + } + + ASSERT(pDataCol->colId == colId); + + while (ccol < pCompBlock->numOfCols) { + pCompCol = &pHelper->pCompData->cols[ccol]; + if (pCompCol->colId >= colId) break; + ccol++; + } + + if (ccol >= pCompBlock->numOfCols || pCompCol->colId > colId) { + dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints); + dcol++; + continue; + } + + ASSERT(pCompCol->colId == pDataCol->colId); + + if (tsdbLoadColData(pHelper, pFile, pCompBlock, pCompCol, pDataCol) < 0) goto _err; + dcol++; + ccol++; + } + + return 0; + +_err: + return -1; +} + +static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) { + ASSERT(pCompBlock->numOfSubBlocks <= 1); SFile *pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF); @@ -1262,6 +1330,8 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa goto _err; } + SCompData *pCompData = (SCompData *)pHelper->pBuffer; + int fd = pFile->fd; if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) { tsdbError("vgId:%d tid:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, @@ -1277,7 +1347,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa } ASSERT(pCompData->numOfCols == pCompBlock->numOfCols); - int32_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM); + int32_t tsize = TSDB_GET_COMPCOL_LEN(pCompBlock->numOfCols); if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) { tsdbError("vgId:%d file %s block data is corrupted offset %" PRId64 " len %d", REPO_ID(pHelper->pRepo), pFile->fname, (int64_t)(pCompBlock->offset), pCompBlock->len); @@ -1315,8 +1385,11 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa } if (tsdbCheckAndDecodeColumnData(pDataCol, (char *)pCompData + tsize + pCompCol->offset, pCompCol->len, pCompBlock->algorithm, pCompBlock->numOfRows, pDataCols->maxPoints, - pHelper->compBuffer, tsizeof(pHelper->compBuffer)) < 0) + pHelper->compBuffer, tsizeof(pHelper->compBuffer)) < 0) { + tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pHelper->pRepo), pFile->fname, + pCompCol->colId, (int64_t)pCompCol->offset); goto _err; + } dcol++; ccol++; } else if (pCompCol->colId < pDataCol->colId) { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 7135c050d5..66131503fb 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -599,7 +599,7 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(pCheckInfo->pTableObj)); - if (tsdbLoadBlockData(&(pQueryHandle->rhelper), pBlock, NULL) == 0) { + if (tsdbLoadBlockData(&(pQueryHandle->rhelper), pBlock) == 0) { SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup; From 3a699e940cca8a12ccddc73af51e95ffc742c769 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 28 Jun 2020 17:42:48 +0800 Subject: [PATCH 07/64] [td-225] handle the error in query --- src/vnode/src/vnodeRead.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 013f8c85ac..7c40364e9c 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -110,19 +110,24 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { pRet->rsp = pRsp; // current connect is broken - if ((code == TSDB_CODE_SUCCESS) && - (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, pQInfo, pVnode->vgId) != TSDB_CODE_SUCCESS)) { - vError("vgId:%d, QInfo:%p, dnode query discarded since link is broken, %p", pVnode->vgId, pQInfo, pReadMsg->rpcMsg.handle); - pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (code == TSDB_CODE_SUCCESS) { + if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, pQInfo, pVnode->vgId) != TSDB_CODE_SUCCESS) { + vError("vgId:%d, QInfo:%p, dnode query discarded since link is broken, %p", pVnode->vgId, pQInfo, + pReadMsg->rpcMsg.handle); + pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - //NOTE: there two refcount, needs to kill twice, todo refactor - qKillQuery(pQInfo, vnodeRelease, pVnode); - qKillQuery(pQInfo, vnodeRelease, pVnode); + // NOTE: there two refcount, needs to kill twice, todo refactor + qKillQuery(pQInfo, vnodeRelease, pVnode); + qKillQuery(pQInfo, vnodeRelease, pVnode); - return pRsp->code; + return pRsp->code; + } + + vTrace("vgId:%d, QInfo:%p, dnode query msg disposed", pVnode->vgId, pQInfo); + } else { + assert(pQInfo == NULL); + vnodeRelease(pVnode); } - - vTrace("vgId:%d, QInfo:%p, dnode query msg disposed", pVnode->vgId, pQInfo); } else { assert(pCont != NULL); pQInfo = pCont; From 51439cb76ff4f20a4cb8cb0f23116b559df0d231 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Sun, 28 Jun 2020 18:18:48 +0800 Subject: [PATCH 08/64] add assert as condition should not be true --- src/client/src/tscLocalMerge.c | 1 + src/client/src/tscSubquery.c | 1 + 2 files changed, 2 insertions(+) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index db85b7a5e5..fba685e703 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -316,6 +316,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->finalRowSize = tscGetResRowLength(pQueryInfo->exprList); pReducer->resColModel = finalmodel; pReducer->resColModel->capacity = pReducer->nResultBufSize; + assert(pReducer->finalRowSize > 0); if (pReducer->finalRowSize > 0) { pReducer->resColModel->capacity /= pReducer->finalRowSize; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 32b8129d02..6e94fd5635 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -936,6 +936,7 @@ static SJoinSupporter* tscUpdateSubqueryStatus(SSqlObj* pSql, int32_t numOfFetch } } + assert(pState != NULL); if (pState != NULL) { pState->numOfTotal = pSql->numOfSubs; pState->numOfRemain = numOfFetch; From 6ef580b4bf3a8a8bc0a1155ba69e453d7cb9c8f6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 28 Jun 2020 11:07:24 +0000 Subject: [PATCH 09/64] [TD-777] use multi-threads to open vnodes --- src/dnode/src/dnodeMgmt.c | 86 ++++++++++++++++++++++++++++++++++----- 1 file changed, 76 insertions(+), 10 deletions(-) diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index a0c23689d0..37244b6cfd 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -39,6 +39,15 @@ #define MPEER_CONTENT_LEN 2000 +typedef struct { + pthread_t thread; + int32_t threadIndex; + int32_t failed; + int32_t opened; + int32_t vnodeNum; + int32_t * vnodeList; +} SOpenVnodeThread; + void * tsDnodeTmr = NULL; static void * tsStatusTimer = NULL; static uint32_t tsRebootTime; @@ -242,14 +251,33 @@ static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { return TSDB_CODE_SUCCESS; } -static int32_t dnodeOpenVnodes() { +static void *dnodeOpenVnode(void *param) { + SOpenVnodeThread *pThread = param; char vnodeDir[TSDB_FILENAME_LEN * 3]; - int32_t failed = 0; - int32_t *vnodeList = (int32_t *)malloc(sizeof(int32_t) * TSDB_MAX_VNODES); - int32_t numOfVnodes; - int32_t status; - status = dnodeGetVnodeList(vnodeList, &numOfVnodes); + dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); + + for (int32_t v = 0; v < pThread->vnodeNum; ++v) { + int32_t vgId = pThread->vnodeList[v]; + snprintf(vnodeDir, TSDB_FILENAME_LEN * 3, "%s/vnode%d", tsVnodeDir, vgId); + if (vnodeOpen(vgId, vnodeDir) < 0) { + dError("vgId:%d, failed to open vnode by thread:%d", vgId, pThread->threadIndex); + pThread->failed++; + } else { + dDebug("vgId:%d, is openned by thread:%d", vgId, pThread->threadIndex); + pThread->opened++; + } + } + + dDebug("thread:%d, total vnodes:%d, openned:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, + pThread->failed); + return NULL; +} + +static int32_t dnodeOpenVnodes() { + int32_t *vnodeList = calloc(TSDB_MAX_VNODES, sizeof(int32_t)); + int32_t numOfVnodes; + int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes); if (status != TSDB_CODE_SUCCESS) { dInfo("Get dnode list failed"); @@ -257,13 +285,51 @@ static int32_t dnodeOpenVnodes() { return status; } - for (int32_t i = 0; i < numOfVnodes; ++i) { - snprintf(vnodeDir, TSDB_FILENAME_LEN * 3, "%s/vnode%d", tsVnodeDir, vnodeList[i]); - if (vnodeOpen(vnodeList[i], vnodeDir) < 0) failed++; + int32_t threadNum = tsNumOfCores; + int32_t vnodesPerThread = numOfVnodes / threadNum + 1; + SOpenVnodeThread *threads = calloc(threadNum, sizeof(SOpenVnodeThread)); + for (int32_t t = 0; t < threadNum; ++t) { + threads[t].threadIndex = t; + threads[t].vnodeList = calloc(vnodesPerThread, sizeof(int32_t)); + } + + for (int32_t v = 0; v < numOfVnodes; ++v) { + int32_t t = v % threadNum; + SOpenVnodeThread *pThread = &threads[t]; + pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v]; + } + + dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); + + for (int32_t t = 0; t < threadNum; ++t) { + SOpenVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum == 0) continue; + + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnode, pThread) != 0) { + dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); + } + + pthread_attr_destroy(&thAttr); + } + + int32_t openVnodes = 0; + int32_t failedVnodes = 0; + for (int32_t t = 0; t < threadNum; ++t) { + SOpenVnodeThread *pThread = &threads[t]; + if (pThread->thread) { + pthread_join(pThread->thread, NULL); + } + openVnodes += pThread->opened; + failedVnodes += pThread->failed; + free(pThread->vnodeList); } free(vnodeList); - dInfo("there are total vnodes:%d, openned:%d failed:%d", numOfVnodes, numOfVnodes-failed, failed); + dInfo("there are total vnodes:%d, openned:%d failed:%d", numOfVnodes, openVnodes, failedVnodes); + return TSDB_CODE_SUCCESS; } From b00b22bb3d58fa6cf62127c17c4b8772902ae684 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 28 Jun 2020 19:11:59 +0800 Subject: [PATCH 10/64] fix file descriptor leakage --- src/tsdb/src/tsdbRWHelper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 0ff67d77d9..2338df1f97 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -163,8 +163,8 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { if (pHelper->files.lastF.fd > 0) { if (helperType(pHelper) == TSDB_WRITE_HELPER) { fsync(pHelper->files.lastF.fd); - close(pHelper->files.lastF.fd); } + close(pHelper->files.lastF.fd); pHelper->files.lastF.fd = -1; } if (helperType(pHelper) == TSDB_WRITE_HELPER) { From 2873422f4f2f5af84b2057e0f3c0ee879756809e Mon Sep 17 00:00:00 2001 From: Hui Li Date: Sun, 28 Jun 2020 19:13:40 +0800 Subject: [PATCH 11/64] [coverity scan] --- src/util/src/ttime.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/util/src/ttime.c b/src/util/src/ttime.c index 176f9be7fb..96e0eaa214 100644 --- a/src/util/src/ttime.c +++ b/src/util/src/ttime.c @@ -56,11 +56,13 @@ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0, year -= 1; } - int64_t res = 367*((int64_t)mon)/12; - - res += ((int64_t)(year/4 - year/100 + year/400 + day + year*365) - 719499); // this value may be less than 0 + //int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + 367*mon/12 + day) + + // year*365 - 719499)*24 + hour)*60 + min)*60 + sec); + int64_t res; + res = 367*((int64_t)mon)/12; + res += year/4 - year/100 + year/400 + day + year*365 - 719499; res = res*24; - res = ((res + hour) * 60 + min) * 60 + sec; + res = ((res + hour) * 60 + min) * 60 + sec; return (res + timezone); } From 8041410772155720f7e512b57548d6a9caac1585 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 28 Jun 2020 14:29:23 +0000 Subject: [PATCH 12/64] [TD-777] --- src/dnode/src/dnodeMgmt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 37244b6cfd..d4032523b1 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -319,7 +319,7 @@ static int32_t dnodeOpenVnodes() { int32_t failedVnodes = 0; for (int32_t t = 0; t < threadNum; ++t) { SOpenVnodeThread *pThread = &threads[t]; - if (pThread->thread) { + if (pThread->vnodeNum > 0 && pThread->thread) { pthread_join(pThread->thread, NULL); } openVnodes += pThread->opened; From 977c855fab4e899747bb9baea3bb2c5d06371fd5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 28 Jun 2020 23:11:14 +0800 Subject: [PATCH 13/64] [td-225] remove print --- src/query/src/qUtil.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 622ec78ade..d7725f561d 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -133,7 +133,6 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { } pWindowResInfo->size = remain; - printf("---------------size:%ld\n", taosHashGetSize(pWindowResInfo->hashList)); for (int32_t k = 0; k < pWindowResInfo->size; ++k) { SWindowResult *pResult = &pWindowResInfo->pResult[k]; int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->window.skey, From 03aa64e9829f693f2e3ce1c5426d401294d22978 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 29 Jun 2020 08:20:33 +0800 Subject: [PATCH 14/64] revert open vnodes codes --- src/dnode/src/dnodeMgmt.c | 86 +++++---------------------------------- 1 file changed, 10 insertions(+), 76 deletions(-) diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index d4032523b1..a0c23689d0 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -39,15 +39,6 @@ #define MPEER_CONTENT_LEN 2000 -typedef struct { - pthread_t thread; - int32_t threadIndex; - int32_t failed; - int32_t opened; - int32_t vnodeNum; - int32_t * vnodeList; -} SOpenVnodeThread; - void * tsDnodeTmr = NULL; static void * tsStatusTimer = NULL; static uint32_t tsRebootTime; @@ -251,33 +242,14 @@ static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { return TSDB_CODE_SUCCESS; } -static void *dnodeOpenVnode(void *param) { - SOpenVnodeThread *pThread = param; - char vnodeDir[TSDB_FILENAME_LEN * 3]; - - dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); - - for (int32_t v = 0; v < pThread->vnodeNum; ++v) { - int32_t vgId = pThread->vnodeList[v]; - snprintf(vnodeDir, TSDB_FILENAME_LEN * 3, "%s/vnode%d", tsVnodeDir, vgId); - if (vnodeOpen(vgId, vnodeDir) < 0) { - dError("vgId:%d, failed to open vnode by thread:%d", vgId, pThread->threadIndex); - pThread->failed++; - } else { - dDebug("vgId:%d, is openned by thread:%d", vgId, pThread->threadIndex); - pThread->opened++; - } - } - - dDebug("thread:%d, total vnodes:%d, openned:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, - pThread->failed); - return NULL; -} - static int32_t dnodeOpenVnodes() { - int32_t *vnodeList = calloc(TSDB_MAX_VNODES, sizeof(int32_t)); + char vnodeDir[TSDB_FILENAME_LEN * 3]; + int32_t failed = 0; + int32_t *vnodeList = (int32_t *)malloc(sizeof(int32_t) * TSDB_MAX_VNODES); int32_t numOfVnodes; - int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes); + int32_t status; + + status = dnodeGetVnodeList(vnodeList, &numOfVnodes); if (status != TSDB_CODE_SUCCESS) { dInfo("Get dnode list failed"); @@ -285,51 +257,13 @@ static int32_t dnodeOpenVnodes() { return status; } - int32_t threadNum = tsNumOfCores; - int32_t vnodesPerThread = numOfVnodes / threadNum + 1; - SOpenVnodeThread *threads = calloc(threadNum, sizeof(SOpenVnodeThread)); - for (int32_t t = 0; t < threadNum; ++t) { - threads[t].threadIndex = t; - threads[t].vnodeList = calloc(vnodesPerThread, sizeof(int32_t)); - } - - for (int32_t v = 0; v < numOfVnodes; ++v) { - int32_t t = v % threadNum; - SOpenVnodeThread *pThread = &threads[t]; - pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v]; - } - - dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); - - for (int32_t t = 0; t < threadNum; ++t) { - SOpenVnodeThread *pThread = &threads[t]; - if (pThread->vnodeNum == 0) continue; - - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnode, pThread) != 0) { - dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); - } - - pthread_attr_destroy(&thAttr); - } - - int32_t openVnodes = 0; - int32_t failedVnodes = 0; - for (int32_t t = 0; t < threadNum; ++t) { - SOpenVnodeThread *pThread = &threads[t]; - if (pThread->vnodeNum > 0 && pThread->thread) { - pthread_join(pThread->thread, NULL); - } - openVnodes += pThread->opened; - failedVnodes += pThread->failed; - free(pThread->vnodeList); + for (int32_t i = 0; i < numOfVnodes; ++i) { + snprintf(vnodeDir, TSDB_FILENAME_LEN * 3, "%s/vnode%d", tsVnodeDir, vnodeList[i]); + if (vnodeOpen(vnodeList[i], vnodeDir) < 0) failed++; } free(vnodeList); - dInfo("there are total vnodes:%d, openned:%d failed:%d", numOfVnodes, openVnodes, failedVnodes); - + dInfo("there are total vnodes:%d, openned:%d failed:%d", numOfVnodes, numOfVnodes-failed, failed); return TSDB_CODE_SUCCESS; } From 80e2fe427c1f9f6876e1251ea2c7cf1d9003590c Mon Sep 17 00:00:00 2001 From: Hui Li Date: Mon, 29 Jun 2020 11:12:22 +0800 Subject: [PATCH 15/64] [coverity scan] --- src/util/src/ttime.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/src/ttime.c b/src/util/src/ttime.c index 96e0eaa214..7fb9738ec7 100644 --- a/src/util/src/ttime.c +++ b/src/util/src/ttime.c @@ -60,7 +60,7 @@ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0, // year*365 - 719499)*24 + hour)*60 + min)*60 + sec); int64_t res; res = 367*((int64_t)mon)/12; - res += year/4 - year/100 + year/400 + day + year*365 - 719499; + res += year/4 - year/100 + year/400 + day + ((int64_t)year)*365 - 719499; res = res*24; res = ((res + hour) * 60 + min) * 60 + sec; From 8a722f7208d1eb1f3003de16628538d97bef030a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 29 Jun 2020 11:51:09 +0800 Subject: [PATCH 16/64] [TD-777] add lock while open vnodes --- src/dnode/src/dnodeMgmt.c | 92 +++++++++++++++++++++++++++++++------ src/dnode/src/dnodeVRead.c | 20 +++++--- src/dnode/src/dnodeVWrite.c | 22 +++++++-- tests/script/test.sh | 2 +- 4 files changed, 111 insertions(+), 25 deletions(-) diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index a0c23689d0..3706f3b6b3 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -39,6 +39,15 @@ #define MPEER_CONTENT_LEN 2000 +typedef struct { + pthread_t thread; + int32_t threadIndex; + int32_t failed; + int32_t opened; + int32_t vnodeNum; + int32_t * vnodeList; +} SOpenVnodeThread; + void * tsDnodeTmr = NULL; static void * tsStatusTimer = NULL; static uint32_t tsRebootTime; @@ -242,28 +251,85 @@ static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { return TSDB_CODE_SUCCESS; } -static int32_t dnodeOpenVnodes() { +static void *dnodeOpenVnode(void *param) { + SOpenVnodeThread *pThread = param; char vnodeDir[TSDB_FILENAME_LEN * 3]; - int32_t failed = 0; - int32_t *vnodeList = (int32_t *)malloc(sizeof(int32_t) * TSDB_MAX_VNODES); - int32_t numOfVnodes; - int32_t status; - status = dnodeGetVnodeList(vnodeList, &numOfVnodes); + dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); + + for (int32_t v = 0; v < pThread->vnodeNum; ++v) { + int32_t vgId = pThread->vnodeList[v]; + snprintf(vnodeDir, TSDB_FILENAME_LEN * 3, "%s/vnode%d", tsVnodeDir, vgId); + if (vnodeOpen(vgId, vnodeDir) < 0) { + dError("vgId:%d, failed to open vnode by thread:%d", vgId, pThread->threadIndex); + pThread->failed++; + } else { + dDebug("vgId:%d, is openned by thread:%d", vgId, pThread->threadIndex); + pThread->opened++; + } + } + + dDebug("thread:%d, total vnodes:%d, openned:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, + pThread->failed); + return NULL; +} + +static int32_t dnodeOpenVnodes() { + int32_t *vnodeList = calloc(TSDB_MAX_VNODES, sizeof(int32_t)); + int32_t numOfVnodes; + int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes); if (status != TSDB_CODE_SUCCESS) { - dInfo("Get dnode list failed"); + dInfo("get dnode list failed"); free(vnodeList); return status; } - for (int32_t i = 0; i < numOfVnodes; ++i) { - snprintf(vnodeDir, TSDB_FILENAME_LEN * 3, "%s/vnode%d", tsVnodeDir, vnodeList[i]); - if (vnodeOpen(vnodeList[i], vnodeDir) < 0) failed++; + int32_t threadNum = tsNumOfCores; + int32_t vnodesPerThread = numOfVnodes / threadNum + 1; + SOpenVnodeThread *threads = calloc(threadNum, sizeof(SOpenVnodeThread)); + for (int32_t t = 0; t < threadNum; ++t) { + threads[t].threadIndex = t; + threads[t].vnodeList = calloc(vnodesPerThread, sizeof(int32_t)); + } + + for (int32_t v = 0; v < numOfVnodes; ++v) { + int32_t t = v % threadNum; + SOpenVnodeThread *pThread = &threads[t]; + pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v]; + } + + dDebug("start %d threads to open %d vnodes", threadNum, numOfVnodes); + + for (int32_t t = 0; t < threadNum; ++t) { + SOpenVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum == 0) continue; + + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnode, pThread) != 0) { + dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); + } + + pthread_attr_destroy(&thAttr); + } + + int32_t openVnodes = 0; + int32_t failedVnodes = 0; + for (int32_t t = 0; t < threadNum; ++t) { + SOpenVnodeThread *pThread = &threads[t]; + if (pThread->vnodeNum > 0 && pThread->thread) { + pthread_join(pThread->thread, NULL); + } + openVnodes += pThread->opened; + failedVnodes += pThread->failed; + free(pThread->vnodeList); } free(vnodeList); - dInfo("there are total vnodes:%d, openned:%d failed:%d", numOfVnodes, numOfVnodes-failed, failed); + dInfo("there are total vnodes:%d, openned:%d failed:%d", numOfVnodes, openVnodes, failedVnodes); + return TSDB_CODE_SUCCESS; } @@ -273,7 +339,7 @@ void dnodeStartStream() { int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes); if (status != TSDB_CODE_SUCCESS) { - dInfo("Get dnode list failed"); + dInfo("get dnode list failed"); return; } @@ -292,7 +358,7 @@ static void dnodeCloseVnodes() { status = dnodeGetVnodeList(vnodeList, &numOfVnodes); if (status != TSDB_CODE_SUCCESS) { - dInfo("Get dnode list failed"); + dInfo("get dnode list failed"); free(vnodeList); return; } diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index 19be83f36b..6bbb291b6a 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -36,6 +36,7 @@ typedef struct { int32_t min; // min number of workers int32_t num; // current number of workers SReadWorker *readWorker; + pthread_mutex_t mutex; } SReadWorkerPool; static void *dnodeProcessReadQueue(void *param); @@ -51,27 +52,28 @@ int32_t dnodeInitVnodeRead() { readPool.min = 2; readPool.max = tsNumOfCores * tsNumOfThreadsPerCore; if (readPool.max <= readPool.min * 2) readPool.max = 2 * readPool.min; - readPool.readWorker = (SReadWorker *) calloc(sizeof(SReadWorker), readPool.max); + readPool.readWorker = (SReadWorker *)calloc(sizeof(SReadWorker), readPool.max); + pthread_mutex_init(&readPool.mutex, NULL); if (readPool.readWorker == NULL) return -1; - for (int i=0; i < readPool.max; ++i) { + for (int i = 0; i < readPool.max; ++i) { SReadWorker *pWorker = readPool.readWorker + i; pWorker->workerId = i; } - dInfo("dnode read is opened"); + dInfo("dnode read is opened, min worker:%d max worker:%d", readPool.min, readPool.max); return 0; } void dnodeCleanupVnodeRead() { - for (int i=0; i < readPool.max; ++i) { + for (int i = 0; i < readPool.max; ++i) { SReadWorker *pWorker = readPool.readWorker + i; if (pWorker->thread) { taosQsetThreadResume(readQset); } } - for (int i=0; i < readPool.max; ++i) { + for (int i = 0; i < readPool.max; ++i) { SReadWorker *pWorker = readPool.readWorker + i; if (pWorker->thread) { pthread_join(pWorker->thread, NULL); @@ -80,6 +82,7 @@ void dnodeCleanupVnodeRead() { free(readPool.readWorker); taosCloseQset(readQset); + pthread_mutex_destroy(&readPool.mutex); dInfo("dnode read is closed"); } @@ -136,8 +139,12 @@ void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg) { } void *dnodeAllocateVnodeRqueue(void *pVnode) { + pthread_mutex_lock(&readPool.mutex); taos_queue queue = taosOpenQueue(); - if (queue == NULL) return NULL; + if (queue == NULL) { + pthread_mutex_unlock(&readPool.mutex); + return NULL; + } taosAddIntoQset(readQset, queue, pVnode); @@ -160,6 +167,7 @@ void *dnodeAllocateVnodeRqueue(void *pVnode) { } while (readPool.num < readPool.min); } + pthread_mutex_unlock(&readPool.mutex); dDebug("pVnode:%p, read queue:%p is allocated", pVnode, queue); return queue; diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index 6de6fceb2f..b20e6c9749 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -47,6 +47,7 @@ typedef struct { int32_t max; // max number of workers int32_t nextId; // from 0 to max-1, cyclic SWriteWorker *writeWorker; + pthread_mutex_t mutex; } SWriteWorkerPool; static void *dnodeProcessWriteQueue(void *param); @@ -58,25 +59,26 @@ int32_t dnodeInitVnodeWrite() { wWorkerPool.max = tsNumOfCores; wWorkerPool.writeWorker = (SWriteWorker *)calloc(sizeof(SWriteWorker), wWorkerPool.max); if (wWorkerPool.writeWorker == NULL) return -1; + pthread_mutex_init(&wWorkerPool.mutex, NULL); for (int32_t i = 0; i < wWorkerPool.max; ++i) { wWorkerPool.writeWorker[i].workerId = i; } - dInfo("dnode write is opened"); + dInfo("dnode write is opened, max worker %d", wWorkerPool.max); return 0; } void dnodeCleanupVnodeWrite() { for (int32_t i = 0; i < wWorkerPool.max; ++i) { - SWriteWorker *pWorker = wWorkerPool.writeWorker + i; + SWriteWorker *pWorker = wWorkerPool.writeWorker + i; if (pWorker->thread) { taosQsetThreadResume(pWorker->qset); } } - + for (int32_t i = 0; i < wWorkerPool.max; ++i) { - SWriteWorker *pWorker = wWorkerPool.writeWorker + i; + SWriteWorker *pWorker = wWorkerPool.writeWorker + i; if (pWorker->thread) { pthread_join(pWorker->thread, NULL); taosFreeQall(pWorker->qall); @@ -84,6 +86,7 @@ void dnodeCleanupVnodeWrite() { } } + pthread_mutex_destroy(&wWorkerPool.mutex); free(wWorkerPool.writeWorker); dInfo("dnode write is closed"); } @@ -124,14 +127,19 @@ void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg) { } void *dnodeAllocateVnodeWqueue(void *pVnode) { + pthread_mutex_lock(&wWorkerPool.mutex); SWriteWorker *pWorker = wWorkerPool.writeWorker + wWorkerPool.nextId; void *queue = taosOpenQueue(); - if (queue == NULL) return NULL; + if (queue == NULL) { + pthread_mutex_unlock(&wWorkerPool.mutex); + return NULL; + } if (pWorker->qset == NULL) { pWorker->qset = taosOpenQset(); if (pWorker->qset == NULL) { taosCloseQueue(queue); + pthread_mutex_unlock(&wWorkerPool.mutex); return NULL; } @@ -140,6 +148,7 @@ void *dnodeAllocateVnodeWqueue(void *pVnode) { if (pWorker->qall == NULL) { taosCloseQset(pWorker->qset); taosCloseQueue(queue); + pthread_mutex_unlock(&wWorkerPool.mutex); return NULL; } pthread_attr_t thAttr; @@ -163,6 +172,7 @@ void *dnodeAllocateVnodeWqueue(void *pVnode) { wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max; } + pthread_mutex_unlock(&wWorkerPool.mutex); dDebug("pVnode:%p, write queue:%p is allocated", pVnode, queue); return queue; @@ -201,6 +211,8 @@ static void *dnodeProcessWriteQueue(void *param) { int type; void *pVnode, *item; + dDebug("write worker:%d is running", pWorker->workerId); + while (1) { numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, pWorker->qall, &pVnode); if (numOfMsgs == 0) { diff --git a/tests/script/test.sh b/tests/script/test.sh index cba0a0fe4b..d9a738749f 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -112,7 +112,7 @@ echo "numOfLogLines 100000000" >> $TAOS_CFG echo "dDebugFlag 135" >> $TAOS_CFG echo "mDebugFlag 135" >> $TAOS_CFG echo "sdbDebugFlag 135" >> $TAOS_CFG -echo "rpcDebugFlag 143" >> $TAOS_CFG +echo "rpcDebugFlag 135" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG echo "cDebugFlag 135" >> $TAOS_CFG echo "httpDebugFlag 135" >> $TAOS_CFG From 71940f706c731b883dc5e4ee2176efd9cc08e2bb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 29 Jun 2020 04:09:21 +0000 Subject: [PATCH 17/64] fix fd leakage --- src/tsdb/src/tsdbMemTable.c | 2 ++ src/tsdb/src/tsdbMeta.c | 14 ++++++-------- src/util/src/tkvstore.c | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index a8ec3b74b4..12db79182e 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -443,12 +443,14 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) { if (tdUpdateKVStoreRecord(pMeta->pStore, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) { tsdbError("vgId:%d failed to update meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, tstrerror(terrno)); + tdKVStoreEndCommit(pMeta->pStore); goto _err; } } else if (pAct->act == TSDB_DROP_META) { if (tdDropKVStoreRecord(pMeta->pStore, pAct->uid) < 0) { tsdbError("vgId:%d failed to drop meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, tstrerror(terrno)); + tdKVStoreEndCommit(pMeta->pStore); goto _err; } } else { diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index a31cb7efef..b769a87beb 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -480,13 +480,11 @@ int tsdbUpdateTable(STsdbRepo *pRepo, STable *pTable, STableCfg *pCfg) { bool changed = false; STsdbMeta *pMeta = pRepo->tsdbMeta; - if (pTable->type == TSDB_SUPER_TABLE) { - if (schemaVersion(pTable->tagSchema) < schemaVersion(pCfg->tagSchema)) { - if (tsdbUpdateTableTagSchema(pTable, pCfg->tagSchema) < 0) { - tsdbError("vgId:%d failed to update table %s tag schema since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - tstrerror(terrno)); - return -1; - } + if ((pTable->type == TSDB_SUPER_TABLE) && (schemaVersion(pTable->tagSchema) < schemaVersion(pCfg->tagSchema))) { + if (tsdbUpdateTableTagSchema(pTable, pCfg->tagSchema) < 0) { + tsdbError("vgId:%d failed to update table %s tag schema since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), + tstrerror(terrno)); + return -1; } changed = true; } @@ -1215,7 +1213,7 @@ static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable) { while (tSkipListIterNext(pIter)) { STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); ASSERT(TABLE_TYPE(tTable) == TSDB_CHILD_TABLE); - pBuf = tsdbInsertTableAct(pRepo, TSDB_DROP_META, pBuf, pTable); + pBuf = tsdbInsertTableAct(pRepo, TSDB_DROP_META, pBuf, tTable); } tSkipListDestroyIter(pIter); diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 42deb9494d..f33941376f 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -259,6 +259,7 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe } taosHashPut(pStore->map, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); + uDebug("put uid %" PRIu64 " into kvStore %s", uid, pStore->fname); return 0; } @@ -292,6 +293,7 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) { pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); taosHashRemove(pStore->map, (void *)(&uid), sizeof(uid)); + uDebug("drop uid %" PRIu64 " from KV store %s", uid, pStore->fname); return 0; } From 8c9c1b3cc3b51735650deee00919d5691f1cafe5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 29 Jun 2020 12:35:35 +0800 Subject: [PATCH 18/64] [td-714] --- src/query/src/qExecutor.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index e72034eb62..04346967d3 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -12,7 +12,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include #include "os.h" #include "qfill.h" @@ -1708,7 +1707,9 @@ static bool onlyFirstQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSD static bool onlyLastQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSDB_FUNC_LAST, TSDB_FUNC_LAST_DST); } -static void changeExecuteScanOrder(SQuery *pQuery, bool stableQuery) { +static void changeExecuteScanOrder(SQInfo *pQInfo, bool stableQuery) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + // in case of point-interpolation query, use asc order scan char msg[] = "QInfo:%p scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%" PRId64 "-%" PRId64 ", new qrange:%" PRId64 "-%" PRId64; @@ -1757,6 +1758,18 @@ static void changeExecuteScanOrder(SQuery *pQuery, bool stableQuery) { pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + + // todo refactor, add iterator + size_t t = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList); + for(int32_t i = 0; i < t; ++i) { + SArray* p1 = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, i); + + size_t len = taosArrayGetSize(p1); + for(int32_t j = 0; j < len; ++j) { + STableQueryInfo* pTableQueryInfo = (STableQueryInfo*) taosArrayGetP(p1, j); + SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); + } + } } pQuery->order.order = TSDB_ORDER_DESC; From d3f2831bf3b8e12b69f6167f90e422ee59d7c674 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 29 Jun 2020 04:52:25 +0000 Subject: [PATCH 19/64] [TD-784] coverity --- src/mnode/src/mnodeSdb.c | 6 +++--- src/mnode/src/mnodeTable.c | 16 ++++++++-------- src/vnode/src/vnodeMain.c | 5 +++-- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 601745461a..e1d255ee76 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -287,7 +287,7 @@ void sdbUpdateSync() { SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); if (pDnode != NULL) { syncCfg.nodeInfo[index].nodePort = pDnode->dnodePort + TSDB_PORT_SYNC; - strcpy(syncCfg.nodeInfo[index].nodeFqdn, pDnode->dnodeEp); + tstrncpy(syncCfg.nodeInfo[index].nodeFqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN); index++; } @@ -976,11 +976,11 @@ static void *sdbWorkerFp(void *param) { tstrerror(pOper->retCode)); } - dnodeSendRpcMnodeWriteRsp(pOper->pMsg, pOper->retCode); - if (pOper != NULL) { sdbDecRef(pOper->table, pOper->pObj); } + + dnodeSendRpcMnodeWriteRsp(pOper->pMsg, pOper->retCode); } taosFreeQitem(item); } diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 023ae68919..2d9892cf01 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1254,13 +1254,13 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, char * pWrite; int32_t cols = 0; SSuperTableObj *pTable = NULL; - char prefix[20] = {0}; + char prefix[64] = {0}; int32_t prefixLen; SDbObj *pDb = mnodeGetDb(pShow->db); if (pDb == NULL) return 0; - strcpy(prefix, pDb->name); + tstrncpy(prefix, pDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); prefixLen = strlen(prefix); @@ -1558,10 +1558,10 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableO static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; - if (pTable != NULL) { - mDebug("app:%p:%p, table:%s, create table in id:%d, uid:%" PRIu64 ", result:%s", pMsg->rpcMsg.ahandle, pMsg, - pTable->info.tableId, pTable->sid, pTable->uid, tstrerror(code)); - } + assert(pTable); + + mDebug("app:%p:%p, table:%s, create table in id:%d, uid:%" PRIu64 ", result:%s", pMsg->rpcMsg.ahandle, pMsg, + pTable->info.tableId, pTable->sid, pTable->uid, tstrerror(code)); if (code != TSDB_CODE_SUCCESS) return code; @@ -2370,7 +2370,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; char prefix[64] = {0}; - strcpy(prefix, pDb->name); + tstrncpy(prefix, pDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); int32_t prefixLen = strlen(prefix); @@ -2560,7 +2560,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; char prefix[64] = {0}; - strcpy(prefix, pDb->name); + tstrncpy(prefix, pDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); int32_t prefixLen = strlen(prefix); diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 89af7b988d..4693dc02d3 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -601,10 +601,11 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { content = calloc(1, maxLen + 1); if (content == NULL) goto PARSE_OVER; - int len = fread(content, 1, maxLen, fp); + int len = fread(content, 1, maxLen, fp); if (len <= 0) { vError("vgId:%d, failed to read vnode cfg, content is null", pVnode->vgId); free(content); + fclose(fp); return errno; } @@ -649,7 +650,7 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->tsdbCfg.maxTables = maxTables->valueint; - cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); + cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); if (!daysPerFile || daysPerFile->type != cJSON_Number) { vError("vgId:%d, failed to read vnode cfg, daysPerFile not found", pVnode->vgId); goto PARSE_OVER; From 552ef8eeca16d46d7157c434cea5031607a5bcc8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 29 Jun 2020 13:00:51 +0800 Subject: [PATCH 20/64] [td-714] --- src/query/src/qExecutor.c | 188 +++++++++++++++++++------------------- 1 file changed, 96 insertions(+), 92 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 04346967d3..74e546e9cb 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -16,7 +16,6 @@ #include "qfill.h" #include "hash.h" -#include "hashfunc.h" #include "qExecutor.h" #include "qUtil.h" #include "qast.h" @@ -24,7 +23,6 @@ #include "query.h" #include "queryLog.h" #include "taosmsg.h" -#include "tdataformat.h" #include "tlosertree.h" #include "tscUtil.h" // todo move the function to common module #include "tscompression.h" @@ -90,6 +88,9 @@ typedef struct { } SQueryStatusInfo; #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) +#define GET_NUM_OF_TABLEGROUP(q) taosArrayGetSize((q)->tableqinfoGroupInfo.pGroupList) +#define GET_TABLEGROUP(q, _index) ((SArray*) taosArrayGetP((q)->tableqinfoGroupInfo.pGroupList, (_index))) + static void setQueryStatus(SQuery *pQuery, int8_t status); static bool isIntervalQuery(SQuery *pQuery) { return pQuery->intervalTime > 0; } @@ -1707,6 +1708,20 @@ static bool onlyFirstQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSD static bool onlyLastQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSDB_FUNC_LAST, TSDB_FUNC_LAST_DST); } +// todo refactor, add iterator +static void doExchangeTimeWindow(SQInfo* pQInfo) { + size_t t = GET_NUM_OF_TABLEGROUP(pQInfo); + for(int32_t i = 0; i < t; ++i) { + SArray* p1 = GET_TABLEGROUP(pQInfo, i); + + size_t len = taosArrayGetSize(p1); + for(int32_t j = 0; j < len; ++j) { + STableQueryInfo* pTableQueryInfo = (STableQueryInfo*) taosArrayGetP(p1, j); + SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); + } + } +} + static void changeExecuteScanOrder(SQInfo *pQInfo, bool stableQuery) { SQuery* pQuery = pQInfo->runtimeEnv.pQuery; @@ -1749,6 +1764,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, bool stableQuery) { pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + doExchangeTimeWindow(pQInfo); } pQuery->order.order = TSDB_ORDER_ASC; @@ -1758,18 +1774,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, bool stableQuery) { pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - - // todo refactor, add iterator - size_t t = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList); - for(int32_t i = 0; i < t; ++i) { - SArray* p1 = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, i); - - size_t len = taosArrayGetSize(p1); - for(int32_t j = 0; j < len; ++j) { - STableQueryInfo* pTableQueryInfo = (STableQueryInfo*) taosArrayGetP(p1, j); - SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); - } - } + doExchangeTimeWindow(pQInfo); } pQuery->order.order = TSDB_ORDER_DESC; @@ -2502,10 +2507,10 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { int64_t st = taosGetTimestampMs(); int32_t ret = TSDB_CODE_SUCCESS; - int32_t numOfGroups = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList); + int32_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); while (pQInfo->groupIndex < numOfGroups) { - SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, pQInfo->groupIndex); + SArray *group = GET_TABLEGROUP(pQInfo, pQInfo->groupIndex); ret = mergeIntoGroupResultImpl(pQInfo, group); if (ret < 0) { // not enough disk space to save the data into disk return -1; @@ -2538,7 +2543,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { } // check if all results has been sent to client - int32_t numOfGroup = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList); + int32_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo); if (pQInfo->numOfGroupResultPages == 0 && pQInfo->groupIndex == numOfGroup) { pQInfo->tableIndex = pQInfo->tableqinfoGroupInfo.numOfTables; // set query completed return; @@ -2872,10 +2877,10 @@ void disableFuncInReverseScan(SQInfo *pQInfo) { } } - int32_t numOfGroups = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList); + int32_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); for(int32_t i = 0; i < numOfGroups; ++i) { - SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, i); + SArray *group = GET_TABLEGROUP(pQInfo, i); size_t t = taosArrayGetSize(group); for (int32_t j = 0; j < t; ++j) { @@ -3362,7 +3367,7 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; pCtx->resultInfo = &pResult->resultInfo[i]; - if (pCtx->resultInfo->complete) { + if (pCtx->resultInfo->initialized && pCtx->resultInfo->complete) { continue; } @@ -3492,7 +3497,7 @@ static int32_t getNumOfSubset(SQInfo *pQInfo) { if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || (isIntervalQuery(pQuery))) { totalSubset = numOfClosedTimeWindow(&pQInfo->runtimeEnv.windowResInfo); } else { - totalSubset = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList); + totalSubset = GET_NUM_OF_TABLEGROUP(pQInfo); } return totalSubset; @@ -3632,36 +3637,40 @@ void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo * bool queryHasRemainResults(SQueryRuntimeEnv* pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; SFillInfo *pFillInfo = pRuntimeEnv->pFillInfo; - - // todo refactor - if (pQuery->fillType == TSDB_FILL_NONE || (pQuery->fillType != TSDB_FILL_NONE && isPointInterpoQuery(pQuery))) { - assert(pFillInfo == NULL); + + if (pQuery->limit.limit > 0 && pQuery->rec.total >= pQuery->limit.limit) { return false; } - if (pQuery->limit.limit > 0 && pQuery->rec.rows >= pQuery->limit.limit) { - return false; - } + if (pQuery->fillType != TSDB_FILL_NONE) { + // There are results not returned to client yet, so filling operation applied to the remain result is required + // in the first place. + int32_t remain = taosNumOfRemainRows(pFillInfo); + if (remain > 0) { + return true; + } - // There are results not returned to client, fill operation applied to the remain result set in the - // first place is required. - int32_t remain = taosNumOfRemainRows(pFillInfo); - if (remain > 0) { - return true; - } - - /* - * While the code reaches here, there are no results returned to client now. - * If query is not completed yet, the gaps between two results blocks need to be handled after next data block - * is retrieved from TSDB. - * - * NOTE: If the result set is not the first block, the gap in front of the result set will be filled. If the result - * set is the FIRST result block, the gap between the start time of query time window and the timestamp of the - * first result row in the actual result set will fill nothing. - */ - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - int32_t numOfTotal = getFilledNumOfRes(pFillInfo, pQuery->window.ekey, pQuery->rec.capacity); - return numOfTotal > 0; + /* + * While the code reaches here, there are no results remains now. + * If query is not completed yet, the gaps between two results blocks need to be handled after next data block + * is retrieved from TSDB. + * + * NOTE: If the result set is not the first block, the gap in front of the result set will be filled. If the result + * set is the FIRST result block, the gap between the start time of query time window and the timestamp of the + * first result row in the actual result set will fill nothing. + */ + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + int32_t numOfTotal = getFilledNumOfRes(pFillInfo, pQuery->window.ekey, pQuery->rec.capacity); + return numOfTotal > 0; + } + + } else { + // there are results waiting for returned to client. + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED) && + (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) && + (pRuntimeEnv->windowResInfo.size > 0)) { + return true; + } } return false; @@ -3703,7 +3712,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data } } -int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int32_t *numOfInterpo) { +int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int32_t *numOfFilled) { SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; SFillInfo* pFillInfo = pRuntimeEnv->pFillInfo; @@ -4008,7 +4017,7 @@ static void setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) { && (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) && (!isFixedOutputQuery(pQuery)) ) { - SArray* pa = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0); + SArray* pa = GET_TABLEGROUP(pQInfo, 0); STableQueryInfo* pCheckInfo = taosArrayGetP(pa, 0); cond.twindow = pCheckInfo->win; } @@ -4052,7 +4061,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool pQuery->precision = tsdbGetCfg(tsdb)->precision; setScanLimitationByResultBuffer(pQuery); - changeExecuteScanOrder(pQuery, false); + changeExecuteScanOrder(pQInfo, false); setupQueryHandle(tsdb, pQInfo, isSTableQuery); pQInfo->tsdb = tsdb; @@ -4155,9 +4164,9 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { STableQueryInfo *pTableQueryInfo = NULL; // todo opt performance using hash table - size_t numOfGroup = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList); + size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo); for (int32_t i = 0; i < numOfGroup; ++i) { - SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, i); + SArray *group = GET_TABLEGROUP(pQInfo, i); size_t num = taosArrayGetSize(group); for (int32_t j = 0; j < num; ++j) { @@ -4210,7 +4219,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { SQuery * pQuery = pRuntimeEnv->pQuery; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0); + SArray *group = GET_TABLEGROUP(pQInfo, 0); STableQueryInfo* pCheckInfo = taosArrayGetP(group, index); setTagVal(pRuntimeEnv, pCheckInfo->pTable, pQInfo->tsdb); @@ -4274,7 +4283,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SQuery * pQuery = pRuntimeEnv->pQuery; setQueryStatus(pQuery, QUERY_COMPLETED); - size_t numOfGroups = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList); + size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); if (isPointInterpoQuery(pQuery) || isFirstLastRowQuery(pQuery)) { resetCtxOutputBuf(pRuntimeEnv); @@ -4324,7 +4333,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { taosArrayDestroy(s); // here we simply set the first table as current table - pQuery->current = (STableQueryInfo*) taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0); + pQuery->current = (STableQueryInfo*) GET_TABLEGROUP(pQInfo, 0); scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey); int64_t numOfRes = getNumOfResult(pRuntimeEnv); @@ -4437,7 +4446,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { resetCtxOutputBuf(pRuntimeEnv); resetTimeWindowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo); - SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0); + SArray *group = GET_TABLEGROUP(pQInfo, 0); assert(taosArrayGetSize(group) == pQInfo->tableqinfoGroupInfo.numOfTables && 1 == taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList)); @@ -4588,9 +4597,9 @@ static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if (isIntervalQuery(pQuery)) { - size_t numOfGroup = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList); + size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo); for (int32_t i = 0; i < numOfGroup; ++i) { - SArray *group = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, i); + SArray *group = GET_TABLEGROUP(pQInfo, i); size_t num = taosArrayGetSize(group); for (int32_t j = 0; j < num; ++j) { @@ -4807,7 +4816,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; pQuery->current = pTableInfo; - int32_t numOfInterpo = 0; + int32_t numOfFilled = 0; TSKEY newStartKey = TSKEY_INITIAL_VAL; // skip blocks without load the actual data block from file if no filter condition present @@ -4835,9 +4844,9 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { } else { taosFillSetStartInfo(pRuntimeEnv->pFillInfo, pQuery->rec.rows, pQuery->window.ekey); taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (tFilePage**) pQuery->sdata); - numOfInterpo = 0; + numOfFilled = 0; - pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfInterpo); + pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled); if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { limitResults(pRuntimeEnv); break; @@ -4856,7 +4865,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { clearFirstNTimeWindow(pRuntimeEnv, pQInfo->groupIndex); } - pQInfo->pointsInterpo += numOfInterpo; + pQInfo->pointsInterpo += numOfFilled; } static void tableQueryImpl(SQInfo *pQInfo) { @@ -4864,45 +4873,40 @@ static void tableQueryImpl(SQInfo *pQInfo) { SQuery * pQuery = pRuntimeEnv->pQuery; if (queryHasRemainResults(pRuntimeEnv)) { - /* - * There are remain results that are not returned due to result interpolation - * So, we do keep in this procedure instead of launching retrieve procedure for next results. - */ - int32_t numOfInterpo = 0; - pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfInterpo); - - if (pQuery->rec.rows > 0) { - limitResults(pRuntimeEnv); - } - - qTrace("QInfo:%p current:%" PRId64 " returned, total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total); - return; - } - // here we have scan all qualified data in both data file and cache - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - // continue to get push data from the group result - if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || - ((isIntervalQuery(pQuery) && pQuery->rec.total < pQuery->limit.limit))) { - // todo limit the output for interval query? + if (pQuery->fillType != TSDB_FILL_NONE) { + /* + * There are remain results that are not returned due to result interpolation + * So, we do keep in this procedure instead of launching retrieve procedure for next results. + */ + int32_t numOfFilled = 0; + pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled); + + if (pQuery->rec.rows > 0) { + limitResults(pRuntimeEnv); + } + + qTrace("QInfo:%p current:%" PRId64 " returned, total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total); + } else { pQuery->rec.rows = 0; pQInfo->groupIndex = 0; // always start from 0 if (pRuntimeEnv->windowResInfo.size > 0) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); - pQuery->rec.rows += pQuery->rec.rows; - clearFirstNTimeWindow(pRuntimeEnv, pQInfo->groupIndex); if (pQuery->rec.rows > 0) { qTrace("QInfo:%p %"PRId64" rows returned from group results, total:%"PRId64"", pQInfo, pQuery->rec.rows, pQuery->rec.total); + + // there are not data remains + if (pRuntimeEnv->windowResInfo.size <= 0) { + qTrace("QInfo:%p query over, %"PRId64" rows are returned", pQInfo, pQuery->rec.total); + } + return; } } } - - qTrace("QInfo:%p query over, %"PRId64" rows are returned", pQInfo, pQuery->rec.total); - return; } // number of points returned during this query @@ -4910,7 +4914,7 @@ static void tableQueryImpl(SQInfo *pQInfo) { int64_t st = taosGetTimestampUs(); assert(pQInfo->tableqinfoGroupInfo.numOfTables == 1); - SArray* g = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0); + SArray* g = GET_TABLEGROUP(pQInfo, 0); STableQueryInfo* item = taosArrayGetP(g, 0); // group by normal column, sliding window query, interval query are handled by interval query processor @@ -5797,9 +5801,9 @@ static void freeQInfo(SQInfo *pQInfo) { } // todo refactor, extract method to destroytableDataInfo - int32_t numOfGroups = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList); + int32_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); for (int32_t i = 0; i < numOfGroups; ++i) { - SArray *p = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, i); + SArray *p = GET_TABLEGROUP(pQInfo, i);; size_t num = taosArrayGetSize(p); for(int32_t j = 0; j < num; ++j) { @@ -6188,14 +6192,14 @@ static void buildTagQueryResult(SQInfo* pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - size_t numOfGroup = taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList); + size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo); assert(numOfGroup == 0 || numOfGroup == 1); if (numOfGroup == 0) { return; } - SArray* pa = taosArrayGetP(pQInfo->tableqinfoGroupInfo.pGroupList, 0); + SArray* pa = GET_TABLEGROUP(pQInfo, 0); size_t num = taosArrayGetSize(pa); assert(num == pQInfo->tableqinfoGroupInfo.numOfTables); From dde667116c4e515371f97046982b1889a0eeb65d Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 29 Jun 2020 13:18:22 +0800 Subject: [PATCH 21/64] TD-775: Remove numpy dependency from sql.py --- ...o-Run-Test-And-How-To-Add-New-Test-Case.md | 2 +- tests/pytest/query/queryJoin.py | 6 +-- tests/pytest/query/querySort.py | 51 +++++++++++++++++-- tests/pytest/util/sql.py | 43 +--------------- 4 files changed, 49 insertions(+), 53 deletions(-) diff --git a/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md b/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md index b6859827bf..b476c118a7 100644 --- a/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md +++ b/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md @@ -11,7 +11,7 @@ 4. pip install ../src/connector/python/linux/python2 ; pip3 install ../src/connector/python/linux/python3 -5. pip install numpy; pip3 install numpy +5. pip install numpy; pip3 install numpy (numpy is required only if you need to run querySort.py) > Note: Both Python2 and Python3 are currently supported by the Python test > framework. Since Python2 is no longer officially supported by Python Software diff --git a/tests/pytest/query/queryJoin.py b/tests/pytest/query/queryJoin.py index b1a0aaed6b..bfc493580e 100644 --- a/tests/pytest/query/queryJoin.py +++ b/tests/pytest/query/queryJoin.py @@ -77,11 +77,7 @@ class TDTestCase: # join queries tdSql.query( "select * from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id") - tdSql.checkRows(6) - - tdSql.query( - "select * from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id order by ts desc") - tdSql.checkColumnSorted(0, "desc") + tdSql.checkRows(6) tdSql.error( "select ts, pressure, temperature, id, dscrption from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id") diff --git a/tests/pytest/query/querySort.py b/tests/pytest/query/querySort.py index 3858701379..e5d3c8ce1f 100644 --- a/tests/pytest/query/querySort.py +++ b/tests/pytest/query/querySort.py @@ -16,6 +16,7 @@ import taos from util.log import * from util.cases import * from util.sql import * +import numpy as np class TDTestCase: @@ -26,6 +27,46 @@ class TDTestCase: self.rowNum = 10 self.ts = 1537146000000 + def checkColumnSorted(self, col, order): + frame = inspect.stack()[1] + callerModule = inspect.getmodule(frame[0]) + callerFilename = callerModule.__file__ + + if col < 0: + tdLog.exit( + "%s failed: sql:%s, col:%d is smaller than zero" % + (callerFilename, tdSql.sql, col)) + if col > tdSql.queryCols: + tdLog.exit( + "%s failed: sql:%s, col:%d is larger than queryCols:%d" % + (callerFilename, tdSql.sql, col, tdSql.queryCols)) + + matrix = np.array(tdSql.queryResult) + list = matrix[:, 0] + + if order == "" or order.upper() == "ASC": + if all(sorted(list) == list): + tdLog.info( + "sql:%s, column :%d is sorted in accending order as expected" % + (tdSql.sql, col)) + else: + tdLog.exit( + "%s failed: sql:%s, col:%d is not sorted in accesnind order" % + (callerFilename, tdSql.sql, col)) + elif order.upper() == "DESC": + if all(sorted(list, reverse=True) == list): + tdLog.info( + "sql:%s, column :%d is sorted in decending order as expected" % + (tdSql.sql, col)) + else: + tdLog.exit( + "%s failed: sql:%s, col:%d is not sorted in decending order" % + (callerFilename, tdSql.sql, col)) + else: + tdLog.exit( + "%s failed: sql:%s, the order provided for col:%d is not correct" % + (callerFilename, tdSql.sql, col)) + def run(self): tdSql.prepare() @@ -49,11 +90,11 @@ class TDTestCase: print("======= step 2: verify order for each column =========") # sort for timestamp in asc order tdSql.query("select * from st order by ts asc") - tdSql.checkColumnSorted(0, "asc") + self.checkColumnSorted(0, "asc") # sort for timestamp in desc order tdSql.query("select * from st order by ts desc") - tdSql.checkColumnSorted(0, "desc") + self.checkColumnSorted(0, "desc") for i in range(1, 10): tdSql.error("select * from st order by tbcol%d" % i) @@ -63,17 +104,17 @@ class TDTestCase: tdSql.query( "select avg(tbcol1) from st group by tagcol%d order by tagcol%d" % (i, i)) - tdSql.checkColumnSorted(1, "") + self.checkColumnSorted(1, "") tdSql.query( "select avg(tbcol1) from st group by tagcol%d order by tagcol%d asc" % (i, i)) - tdSql.checkColumnSorted(1, "asc") + self.checkColumnSorted(1, "asc") tdSql.query( "select avg(tbcol1) from st group by tagcol%d order by tagcol%d desc" % (i, i)) - tdSql.checkColumnSorted(1, "desc") + self.checkColumnSorted(1, "desc") def stop(self): tdSql.close() diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index e282298b7c..dc0366b214 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -17,7 +17,6 @@ import time import datetime import inspect from util.log import * -import numpy as np class TDSql: @@ -199,47 +198,7 @@ class TDSql: "%s failed: sql:%s, affectedRows:%d != expect:%d" % (callerFilename, self.sql, self.affectedRows, expectAffectedRows)) tdLog.info("sql:%s, affectedRows:%d == expect:%d" % - (self.sql, self.affectedRows, expectAffectedRows)) - - def checkColumnSorted(self, col, order): - frame = inspect.stack()[1] - callerModule = inspect.getmodule(frame[0]) - callerFilename = callerModule.__file__ - - if col < 0: - tdLog.exit( - "%s failed: sql:%s, col:%d is smaller than zero" % - (callerFilename, self.sql, col)) - if col > self.queryCols: - tdLog.exit( - "%s failed: sql:%s, col:%d is larger than queryCols:%d" % - (callerFilename, self.sql, col, self.queryCols)) - - matrix = np.array(self.queryResult) - list = matrix[:, 0] - - if order == "" or order.upper() == "ASC": - if all(sorted(list) == list): - tdLog.info( - "sql:%s, column :%d is sorted in accending order as expected" % - (self.sql, col)) - else: - tdLog.exit( - "%s failed: sql:%s, col:%d is not sorted in accesnind order" % - (callerFilename, self.sql, col)) - elif order.upper() == "DESC": - if all(sorted(list, reverse=True) == list): - tdLog.info( - "sql:%s, column :%d is sorted in decending order as expected" % - (self.sql, col)) - else: - tdLog.exit( - "%s failed: sql:%s, col:%d is not sorted in decending order" % - (callerFilename, self.sql, col)) - else: - tdLog.exit( - "%s failed: sql:%s, the order provided for col:%d is not correct" % - (callerFilename, self.sql, col)) + (self.sql, self.affectedRows, expectAffectedRows)) tdSql = TDSql() From 0102f0a165cda548770e9231f225d2eab4c365a4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 29 Jun 2020 06:02:00 +0000 Subject: [PATCH 22/64] scripts --- tests/script/basicSuite.sim | 7 +++---- tests/script/jenkins/simple.txt | 21 ------------------- tests/script/jenkins/unique.txt | 7 ++++--- tests/script/unique/vnode/replica2_repeat.sim | 2 +- tests/script/unique/vnode/replica3_repeat.sim | 2 +- 5 files changed, 9 insertions(+), 30 deletions(-) diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index de6ed85b09..3701a33d27 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -2,17 +2,16 @@ run general/cache/new_metrics.sim run general/column/commit.sim run general/compress/compress.sim run general/compute/interval.sim -run general/db/basic4.sim -run general/field/binary.sim run general/http/restful_insert.sim run general/import/commit.sim run general/import/replica1.sim run general/parser/auto_create_tb_drop_tb.sim run general/parser/binary_escapeCharacter.sim run general/parser/select_from_cache_disk.sim +run general/parser/join.sim +run general/parser/where.sim +run general/parser/projection_limit_offset.sim run general/stable/vnode3.sim -run general/table/autocreate.sim -run general/table/fill.sim run general/table/vgroup.sim run general/tag/filter.sim run general/user/authority.sim diff --git a/tests/script/jenkins/simple.txt b/tests/script/jenkins/simple.txt index b3716585a9..135326af9e 100644 --- a/tests/script/jenkins/simple.txt +++ b/tests/script/jenkins/simple.txt @@ -232,24 +232,3 @@ cd ../../../debug; make ./test.sh -f general/vector/table_mix.sim ./test.sh -f general/vector/table_query.sim ./test.sh -f general/vector/table_time.sim - -./test.sh -f unique/account/account_create.sim -./test.sh -f unique/account/account_delete.sim -./test.sh -f unique/account/account_len.sim -./test.sh -f unique/account/authority.sim -./test.sh -f unique/account/basic.sim -./test.sh -f unique/account/paras.sim -./test.sh -f unique/account/pass_alter.sim -./test.sh -f unique/account/pass_len.sim -./test.sh -f unique/account/usage.sim -./test.sh -f unique/account/user_create.sim -./test.sh -f unique/account/user_len.sim - -./test.sh -f unique/cluster/balance1.sim -./test.sh -f unique/cluster/balance2.sim -./test.sh -f unique/dnode/balance1.sim -./test.sh -f unique/dnode/balance2.sim -./test.sh -f unique/stable/dnode3.sim -./test.sh -f unique/mnode/mgmt22.sim -./test.sh -f unique/mnode/mgmt33.sim -./test.sh -f unique/vnode/many.sim \ No newline at end of file diff --git a/tests/script/jenkins/unique.txt b/tests/script/jenkins/unique.txt index da0aee3d34..afd0ea55c0 100644 --- a/tests/script/jenkins/unique.txt +++ b/tests/script/jenkins/unique.txt @@ -102,9 +102,10 @@ cd ../../../debug; make ./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim ./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim +#./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim +#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim +#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim +#./test.sh -f unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim ./test.sh -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim ./test.sh -f unique/arbitrator/dn3_mn1_vnode_delDir.sim ./test.sh -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim diff --git a/tests/script/unique/vnode/replica2_repeat.sim b/tests/script/unique/vnode/replica2_repeat.sim index 44a86763ed..e862d745d4 100644 --- a/tests/script/unique/vnode/replica2_repeat.sim +++ b/tests/script/unique/vnode/replica2_repeat.sim @@ -64,7 +64,7 @@ print ======== step7 $lastRows = $data00 print ======== loop Times $x -if $x < 2 then +if $x < 5 then $x = $x + 1 goto loop endi diff --git a/tests/script/unique/vnode/replica3_repeat.sim b/tests/script/unique/vnode/replica3_repeat.sim index 9c3d5bbf09..4b5c852de8 100644 --- a/tests/script/unique/vnode/replica3_repeat.sim +++ b/tests/script/unique/vnode/replica3_repeat.sim @@ -75,7 +75,7 @@ print ======== step8 $lastRows = $data00 print ======== loop Times $x -if $x < 2 then +if $x < 5 then $x = $x + 1 goto loop endi From 19299f27c57dfbc6a42d8f946c52eca66648a4b2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 29 Jun 2020 06:08:24 +0000 Subject: [PATCH 23/64] adjust script for CI --- tests/script/basicSuite.sim | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index 3701a33d27..0d82f96586 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -1,20 +1,18 @@ run general/cache/new_metrics.sim -run general/column/commit.sim -run general/compress/compress.sim run general/compute/interval.sim -run general/http/restful_insert.sim +run general/http/restful_full.sim run general/import/commit.sim run general/import/replica1.sim run general/parser/auto_create_tb_drop_tb.sim run general/parser/binary_escapeCharacter.sim run general/parser/select_from_cache_disk.sim +run general/parser/projection_limit_offset.sim run general/parser/join.sim run general/parser/where.sim run general/parser/projection_limit_offset.sim run general/stable/vnode3.sim -run general/table/vgroup.sim run general/tag/filter.sim +run general/table/vgroup.sim run general/user/authority.sim -run general/user/pass_alter.sim run general/vector/metrics_mix.sim run general/vector/table_field.sim From bee633127c7acdb4cd68092ef10fcde6b2a60fe2 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 29 Jun 2020 14:35:43 +0800 Subject: [PATCH 24/64] remove some pytest cases from smoke test. [TD-786] --- tests/pytest/smoketest.sh | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/tests/pytest/smoketest.sh b/tests/pytest/smoketest.sh index c56a4726dd..6b21912dd5 100755 --- a/tests/pytest/smoketest.sh +++ b/tests/pytest/smoketest.sh @@ -8,24 +8,8 @@ python3 ./test.py $1 -s && sleep 1 # insert python3 ./test.py $1 -f insert/basic.py python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/int.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/float.py -python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f insert/bigint.py python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/bool.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/double.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/smallint.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/tinyint.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/binary.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/date.py -python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f insert/nchar.py python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f insert/multi.py @@ -42,18 +26,6 @@ python3 ./test.py $1 -s && sleep 1 # import python3 ./test.py $1 -f import_merge/importDataLastSub.py python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importHead.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importLastT.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importSpan.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importTail.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importTRestart.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importInsertThenImport.py -python3 ./test.py $1 -s && sleep 1 #tag python3 ./test.py $1 -f tag_lite/filter.py From 3f35736595e209c1937eaa82e3943cee86eba036 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 29 Jun 2020 14:40:57 +0800 Subject: [PATCH 25/64] [td-225] fix bugs in regression test. --- src/query/src/qExecutor.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 18d11c6fa6..e45632cb5b 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4887,6 +4887,7 @@ static void tableQueryImpl(SQInfo *pQInfo) { } qDebug("QInfo:%p current:%" PRId64 " returned, total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total); + return; } else { pQuery->rec.rows = 0; pQInfo->groupIndex = 0; // always start from 0 From c0878b8ca1f5ae12336604314193bb8017df2c43 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 29 Jun 2020 15:14:25 +0800 Subject: [PATCH 26/64] [td-225] fix bugs in regression test. --- src/query/src/qExecutor.c | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index e45632cb5b..6de9b56b3a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5676,26 +5676,6 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, return pQInfo; _cleanup: - //tfree(pQuery->fillVal); - - //if (pQuery->sdata != NULL) { - // for (int16_t col = 0; col < pQuery->numOfOutput; ++col) { - // tfree(pQuery->sdata[col]); - // } - //} - - // - //tfree(pQuery->sdata); - //tfree(pQuery->pFilterInfo); - //tfree(pQuery->colList); - - //tfree(pExprs); - //tfree(pGroupbyExpr); - - //taosArrayDestroy(pQInfo->arrTableIdInfo); - //tsdbDestoryTableGroup(&pQInfo->tableGroupInfo); - // - //tfree(pQInfo); freeQInfo(pQInfo); return NULL; @@ -5730,7 +5710,6 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ UNUSED(ret); } - // only the successful complete requries the sem_post/over = 1 operations. if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { qDebug("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, @@ -5740,7 +5719,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ sem_post(&pQInfo->dataReady); return TSDB_CODE_SUCCESS; } - + if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { qDebug("QInfo:%p no table qualified for tag filter, abort query", pQInfo); setQueryStatus(pQuery, QUERY_COMPLETED); @@ -6071,6 +6050,11 @@ void qTableQuery(qinfo_t qinfo, void (*fp)(void*), void* param) { return; } + if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { + qDebug("QInfo:%p no table exists for query, abort", pQInfo); + return; + } + qDebug("QInfo:%p query task is launched", pQInfo); if (onlyQueryTags(pQInfo->runtimeEnv.pQuery)) { From dc5ce545f28384c2cb793822fae33104f2564eee Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 29 Jun 2020 15:40:40 +0800 Subject: [PATCH 27/64] move more cases to regression. --- tests/pytest/regressiontest.sh | 5 +++-- tests/script/regressionSuite.sim | 18 +++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/pytest/regressiontest.sh b/tests/pytest/regressiontest.sh index 14b18e1acc..8ce1fd1294 100755 --- a/tests/pytest/regressiontest.sh +++ b/tests/pytest/regressiontest.sh @@ -43,7 +43,7 @@ python3 ./test.py -f tag_lite/commit.py python3 ./test.py -f tag_lite/create.py python3 ./test.py -f tag_lite/datatype.py python3 ./test.py -f tag_lite/datatype-without-alter.py -# python3 ./test.py -f tag_lite/delete.py +python3 ./test.py -f tag_lite/delete.py python3 ./test.py -f tag_lite/double.py python3 ./test.py -f tag_lite/float.py python3 ./test.py -f tag_lite/int_binary.py @@ -134,9 +134,10 @@ python3 ./test.py -f table/del_stable.py python3 ./test.py -f query/filter.py python3 ./test.py -f query/filterAllIntTypes.py python3 ./test.py -f query/filterFloatAndDouble.py +python3 ./test.py -f query/filterOtherTypes.py +python3 ./test.py -f query/queryError.py python3 ./test.py -f query/querySort.py - #stream python3 ./test.py -f stream/stream1.py python3 ./test.py -f stream/stream2.py diff --git a/tests/script/regressionSuite.sim b/tests/script/regressionSuite.sim index 0fbec7ebd8..c671cf47c6 100644 --- a/tests/script/regressionSuite.sim +++ b/tests/script/regressionSuite.sim @@ -1,10 +1,10 @@ ##unsupport run general/alter/cached_schema_after_alter.sim -unsupport run general/alter/count.sim -unsupport run general/alter/import.sim +run general/alter/count.sim +run general/alter/import.sim ##unsupport run general/alter/insert1.sim -unsupport run general/alter/insert2.sim -unsupport run general/alter/metrics.sim -unsupport run general/alter/table.sim +run general/alter/insert2.sim +run general/alter/metrics.sim +run general/alter/table.sim run general/cache/new_metrics.sim run general/cache/restart_metrics.sim run general/cache/restart_table.sim @@ -86,14 +86,14 @@ run general/insert/query_block2_file.sim run general/insert/query_file_memory.sim run general/insert/query_multi_file.sim run general/insert/tcp.sim -##unsupport run general/parser/alter.sim +run general/parser/alter.sim run general/parser/alter1.sim run general/parser/alter_stable.sim run general/parser/auto_create_tb.sim run general/parser/auto_create_tb_drop_tb.sim run general/parser/col_arithmetic_operation.sim run general/parser/columnValue.sim -#run general/parser/commit.sim +run general/parser/commit.sim run general/parser/create_db.sim run general/parser/create_mt.sim run general/parser/create_tb.sim @@ -106,7 +106,7 @@ run general/parser/first_last.sim ##unsupport run general/parser/import_file.sim run general/parser/lastrow.sim run general/parser/nchar.sim -##unsupport run general/parser/null_char.sim +run general/parser/null_char.sim run general/parser/single_row_in_tb.sim run general/parser/select_from_cache_disk.sim run general/parser/limit.sim @@ -132,7 +132,7 @@ run general/parser/groupby.sim run general/parser/bug.sim run general/parser/tags_dynamically_specifiy.sim run general/parser/set_tag_vals.sim -##unsupport run general/parser/repeatAlter.sim +run general/parser/repeatAlter.sim ##unsupport run general/parser/slimit_alter_tags.sim ##unsupport run general/parser/stream_on_sys.sim run general/parser/stream.sim From b0713a1d7bbf0dfc3c879cc16005e6287c761372 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 29 Jun 2020 15:46:39 +0800 Subject: [PATCH 28/64] scripts --- tests/script/basicSuite.sim | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index 0d82f96586..f6b45949fb 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -6,10 +6,8 @@ run general/import/replica1.sim run general/parser/auto_create_tb_drop_tb.sim run general/parser/binary_escapeCharacter.sim run general/parser/select_from_cache_disk.sim -run general/parser/projection_limit_offset.sim -run general/parser/join.sim -run general/parser/where.sim -run general/parser/projection_limit_offset.sim +run general/parser/null_char.sim +run general/parser/alter.sim run general/stable/vnode3.sim run general/tag/filter.sim run general/table/vgroup.sim From 02bba66835de75426165f9e4862cce3e895e91f9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 29 Jun 2020 16:48:38 +0800 Subject: [PATCH 29/64] [td-225] fix bugs in regression test. --- src/query/src/qExecutor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6de9b56b3a..6bc55effdc 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3642,7 +3642,7 @@ bool queryHasRemainResults(SQueryRuntimeEnv* pRuntimeEnv) { return false; } - if (pQuery->fillType != TSDB_FILL_NONE) { + if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) { // There are results not returned to client yet, so filling operation applied to the remain result is required // in the first place. int32_t remain = taosNumOfRemainRows(pFillInfo); From d026efa9ca8a9b50d0037e17b1c38a5a3b79b242 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 29 Jun 2020 17:03:32 +0800 Subject: [PATCH 30/64] fix tag_filter invalid read --- src/common/src/tdataformat.c | 2 +- src/query/src/qExecutor.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 5610f0f75e..d7b2af8706 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -259,7 +259,7 @@ bool isNEleNull(SDataCol *pCol, int nEle) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: for (int i = 0; i < nEle; i++) { - if (!isNull(varDataVal(tdGetColDataOfRow(pCol, i)), pCol->type)) return false; + if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false; } return true; default: diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 8f565cd9c6..aeacdd2351 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2219,7 +2219,7 @@ static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVa } if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - if (isNull(varDataVal(val), type)) { + if (isNull(val, type)) { tag->nType = TSDB_DATA_TYPE_NULL; return; } From d7397bce19008f83d5a29a219aafc3f6a759ed26 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 29 Jun 2020 17:11:26 +0800 Subject: [PATCH 31/64] change VM timezone to GMT+8 to satisfy some test cases. [TD-788] --- .travis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.travis.yml b/.travis.yml index bb9ee5d430..48845c1f86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,6 +36,8 @@ matrix: - psmisc before_script: + - echo 'Asia/Harbin' | sudo tee /etc/timezone + - sudo dpkg-reconfigure --frontend --noninteractive tzdata - cd ${TRAVIS_BUILD_DIR} - mkdir debug - cd debug @@ -150,6 +152,8 @@ matrix: - DESC="trusty/gcc-4.8 build" before_script: + - echo 'Asia/Harbin' | sudo tee /etc/timezone + - sudo dpkg-reconfigure --frontend --noninteractive tzdata - cd ${TRAVIS_BUILD_DIR} - mkdir debug - cd debug @@ -173,6 +177,8 @@ matrix: - cmake before_script: + - echo 'Asia/Harbin' | sudo tee /etc/timezone + - sudo dpkg-reconfigure --frontend --noninteractive tzdata - cd ${TRAVIS_BUILD_DIR} - mkdir debug - cd debug @@ -197,6 +203,8 @@ matrix: - cmake before_script: + - echo 'Asia/Harbin' | sudo tee /etc/timezone + - sudo dpkg-reconfigure --frontend --noninteractive tzdata - cd ${TRAVIS_BUILD_DIR} - mkdir debug - cd debug @@ -225,6 +233,8 @@ matrix: - DESC="trusty/gcc-4.8 build" before_script: + - echo 'Asia/Harbin' | sudo tee /etc/timezone + - sudo dpkg-reconfigure --frontend --noninteractive tzdata - cd ${TRAVIS_BUILD_DIR} - mkdir debug - cd debug From 97340bbdf5a500637fa7712d6e4596903f08f48b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 29 Jun 2020 09:19:32 +0000 Subject: [PATCH 32/64] definite lost while open vnodes --- src/dnode/src/dnodeMgmt.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 3706f3b6b3..9cf024ba83 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -328,6 +328,7 @@ static int32_t dnodeOpenVnodes() { } free(vnodeList); + free(threads); dInfo("there are total vnodes:%d, openned:%d failed:%d", numOfVnodes, openVnodes, failedVnodes); return TSDB_CODE_SUCCESS; From 9fce5542e73a30a150bc9e9717a427cd15425372 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 29 Jun 2020 17:39:41 +0800 Subject: [PATCH 33/64] fix commit bug --- src/tsdb/src/tsdbFile.c | 2 +- src/tsdb/src/tsdbRWHelper.c | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index ed5f62d3da..33eae639b8 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -29,7 +29,7 @@ #include "tutil.h" #include "ttime.h" -const char *tsdbFileSuffix[] = {".head", ".data", ".last", "", ".h", ".h"}; +const char *tsdbFileSuffix[] = {".head", ".data", ".last", "", ".h", ".l"}; static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); static void tsdbDestroyFile(SFile *pFile); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 41725433eb..934fa8e733 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -128,11 +128,12 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { // Create and open .l file if should if (tsdbShouldCreateNewLast(pHelper)) { if (tsdbOpenFile(&(pHelper->files.nLastF), O_WRONLY | O_CREAT) < 0) goto _err; - if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) + if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { tsdbError("vgId:%d failed to sendfile %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), TSDB_FILE_HEAD_SIZE, pHelper->files.lastF.fname, pHelper->files.nLastF.fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } } } else { if (tsdbOpenFile(&(pHelper->files.dataF), O_RDONLY) < 0) goto _err; @@ -144,7 +145,7 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { return tsdbLoadCompIdx(pHelper, NULL); _err: - return terrno; + return -1; } int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { From 12eb25514847b2c5954152a73226e6261e878c9d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 29 Jun 2020 17:54:32 +0800 Subject: [PATCH 34/64] fix return code --- src/vnode/src/vnodeMain.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 4693dc02d3..cbd7953072 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -123,9 +123,8 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { char tsdbDir[TSDB_FILENAME_LEN] = {0}; sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); - code = tsdbCreateRepo(tsdbDir, &tsdbCfg); - if (code != TSDB_CODE_SUCCESS) { - vError("vgId:%d, failed to create tsdb in vnode, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(code)); + if (tsdbCreateRepo(tsdbDir, &tsdbCfg) < 0) { + vError("vgId:%d, failed to create tsdb in vnode, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(terrno)); return TSDB_CODE_VND_INIT_FAILED; } From 92a18381726d9a0f5d7344cbb63480ac6068b3a8 Mon Sep 17 00:00:00 2001 From: dengyihao Date: Mon, 29 Jun 2020 18:16:36 +0800 Subject: [PATCH 35/64] change ref-count type --- src/query/src/qExecutor.c | 2 +- src/tsdb/src/tsdbMeta.c | 6 +++--- src/util/inc/tref.h | 8 ++++---- src/util/src/tcache.c | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 94fefa16f3..ff9cff54bf 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6009,7 +6009,7 @@ void qDestroyQueryInfo(qinfo_t qHandle, void (*fp)(void*), void* param) { return; } - int16_t ref = T_REF_DEC(pQInfo); + int32_t ref = T_REF_DEC(pQInfo); qDebug("QInfo:%p dec refCount, value:%d", pQInfo, ref); if (ref == 0) { diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 3b25d68896..13b676ee1d 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -550,13 +550,13 @@ int tsdbUnlockRepoMeta(STsdbRepo *pRepo) { } void tsdbRefTable(STable *pTable) { - int16_t ref = T_REF_INC(pTable); + int32_t ref = T_REF_INC(pTable); UNUSED(ref); // tsdbDebug("ref table %"PRIu64", tid:%d, refCount:%d", TABLE_UID(pTable), TABLE_TID(pTable), ref); } void tsdbUnRefTable(STable *pTable) { - int16_t ref = T_REF_DEC(pTable); + int32_t ref = T_REF_DEC(pTable); tsdbDebug("unref table uid:%"PRIu64", tid:%d, refCount:%d", TABLE_UID(pTable), TABLE_TID(pTable), ref); if (ref == 0) { @@ -1252,4 +1252,4 @@ static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable) { } return 0; -} \ No newline at end of file +} diff --git a/src/util/inc/tref.h b/src/util/inc/tref.h index fa2517e19a..0503325326 100644 --- a/src/util/inc/tref.h +++ b/src/util/inc/tref.h @@ -22,7 +22,7 @@ typedef void (*_ref_fn_t)(const void* pObj); #define T_REF_DECLARE() \ struct { \ - int16_t val; \ + int32_t val; \ } _ref; #define T_REF_REGISTER_FUNC(s, e) \ @@ -31,7 +31,7 @@ typedef void (*_ref_fn_t)(const void* pObj); _ref_fn_t end; \ } _ref_func = {.begin = (s), .end = (e)}; -#define T_REF_INC(x) (atomic_add_fetch_16(&((x)->_ref.val), 1)) +#define T_REF_INC(x) (atomic_add_fetch_32(&((x)->_ref.val), 1)) #define T_REF_INC_WITH_CB(x, p) \ do { \ @@ -41,11 +41,11 @@ typedef void (*_ref_fn_t)(const void* pObj); } \ } while (0) -#define T_REF_DEC(x) (atomic_sub_fetch_16(&((x)->_ref.val), 1)) +#define T_REF_DEC(x) (atomic_sub_fetch_32(&((x)->_ref.val), 1)) #define T_REF_DEC_WITH_CB(x, p) \ do { \ - int32_t v = atomic_sub_fetch_16(&((x)->_ref.val), 1); \ + int32_t v = atomic_sub_fetch_32(&((x)->_ref.val), 1); \ if (v == 0 && (p)->_ref_func.end != NULL) { \ (p)->_ref_func.end((x)); \ } \ diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 2dd641731c..2e57ad83ae 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -415,7 +415,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { } *data = NULL; - int16_t ref = T_REF_DEC(pNode); + int32_t ref = T_REF_DEC(pNode); uDebug("%p data released, refcnt:%d", pNode, ref); if (_remove) { From 43b7996c90e5626e736c6e7efe93fb792ed9731f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 29 Jun 2020 19:12:50 +0800 Subject: [PATCH 36/64] [td-225] opt perf --- src/inc/tsdb.h | 6 ++-- src/query/src/qExecutor.c | 67 +++++++++++++++++++++------------------ 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 7c8a7af570..e2fec9b3dd 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -24,6 +24,7 @@ #include "tarray.h" #include "tdataformat.h" #include "tname.h" +#include "hash.h" #ifdef __cplusplus extern "C" { @@ -168,8 +169,9 @@ typedef struct SDataBlockInfo { } SDataBlockInfo; typedef struct { - size_t numOfTables; - SArray *pGroupList; + size_t numOfTables; + SArray *pGroupList; + SHashObj *map; // speedup acquire the tableQueryInfo from STableId } STableGroupInfo; typedef struct SQueryRowCond { diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6bc55effdc..cbc7d23c0a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4161,33 +4161,36 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { } SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); - STableQueryInfo *pTableQueryInfo = NULL; - - // todo opt performance using hash table - size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo); - for (int32_t i = 0; i < numOfGroup; ++i) { - SArray *group = GET_TABLEGROUP(pQInfo, i); - - size_t num = taosArrayGetSize(group); - for (int32_t j = 0; j < num; ++j) { - STableQueryInfo *p = taosArrayGetP(group, j); - - STableId id = tsdbGetTableId(p->pTable); - if (id.tid == blockInfo.tid) { - assert(id.uid == blockInfo.uid); - pTableQueryInfo = p; - - break; - } - } - - if (pTableQueryInfo != NULL) { - break; - } + STableQueryInfo **pTableQueryInfo = (STableQueryInfo**) taosHashGet(pQInfo->tableqinfoGroupInfo.map, &blockInfo.tid, sizeof(blockInfo.tid)); + if(pTableQueryInfo == NULL) { + break; } - - assert(pTableQueryInfo != NULL); - setCurrentQueryTable(pRuntimeEnv, pTableQueryInfo); + // todo opt performance using hash table + +// size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo); +// for (int32_t i = 0; i < numOfGroup; ++i) { +// SArray *group = GET_TABLEGROUP(pQInfo, i); +// +// size_t num = taosArrayGetSize(group); +// for (int32_t j = 0; j < num; ++j) { +// STableQueryInfo *p = taosArrayGetP(group, j); +// +// STableId id = tsdbGetTableId(p->pTable); +// if (id.tid == blockInfo.tid) { +// assert(id.uid == blockInfo.uid); +// pTableQueryInfo = p; +// +// break; +// } +// } +// +// if (pTableQueryInfo != NULL) { +// break; +// } +// } + + assert(*pTableQueryInfo != NULL); + setCurrentQueryTable(pRuntimeEnv, *pTableQueryInfo); SDataStatis *pStatis = NULL; SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis); @@ -4195,11 +4198,12 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { if (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) { if (!isIntervalQuery(pQuery)) { int32_t step = QUERY_IS_ASC_QUERY(pQuery)? 1:-1; - setExecutionContext(pQInfo, pTableQueryInfo->pTable, pTableQueryInfo->groupIndex, blockInfo.window.ekey + step); + setExecutionContext(pQInfo, (*pTableQueryInfo)->pTable, (*pTableQueryInfo)->groupIndex, + blockInfo.window.ekey + step); } else { // interval query TSKEY nextKey = blockInfo.window.skey; setIntervalQueryRange(pQInfo, nextKey); - /*int32_t ret = */setAdditionalInfo(pQInfo, pTableQueryInfo->pTable, pTableQueryInfo); + /*int32_t ret = */setAdditionalInfo(pQInfo, (*pTableQueryInfo)->pTable, *pTableQueryInfo); } } @@ -5627,7 +5631,9 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, pQInfo->tableqinfoGroupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); pQInfo->tableqinfoGroupInfo.numOfTables = pTableGroupInfo->numOfTables; - } + pQInfo->tableqinfoGroupInfo.map = taosHashInit(pTableGroupInfo->numOfTables, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + } int tableIndex = 0; STimeWindow window = pQueryMsg->window; @@ -5655,6 +5661,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, item->groupIndex = i; item->tableIndex = tableIndex++; taosArrayPush(p1, &item); + taosHashPut(pQInfo->tableqinfoGroupInfo.map, &id.tid, sizeof(id.tid), &item, POINTER_BYTES); } taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1); @@ -5797,7 +5804,7 @@ static void freeQInfo(SQInfo *pQInfo) { } taosArrayDestroy(pQInfo->tableqinfoGroupInfo.pGroupList); - + taosHashCleanup(pQInfo->tableqinfoGroupInfo.map); tsdbDestoryTableGroup(&pQInfo->tableGroupInfo); taosArrayDestroy(pQInfo->arrTableIdInfo); From a2bf3af34b1b3e9dc2956d2028ea3b8678f18e98 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 29 Jun 2020 19:33:50 +0800 Subject: [PATCH 37/64] change way to set timezone. --- .travis.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 48845c1f86..0256df7907 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,8 +36,7 @@ matrix: - psmisc before_script: - - echo 'Asia/Harbin' | sudo tee /etc/timezone - - sudo dpkg-reconfigure --frontend --noninteractive tzdata + - sudo timedatectl set-timezone Asia/Harbin - cd ${TRAVIS_BUILD_DIR} - mkdir debug - cd debug @@ -152,8 +151,7 @@ matrix: - DESC="trusty/gcc-4.8 build" before_script: - - echo 'Asia/Harbin' | sudo tee /etc/timezone - - sudo dpkg-reconfigure --frontend --noninteractive tzdata + - sudo timedatectl set-timezone Asia/Harbin - cd ${TRAVIS_BUILD_DIR} - mkdir debug - cd debug @@ -177,8 +175,7 @@ matrix: - cmake before_script: - - echo 'Asia/Harbin' | sudo tee /etc/timezone - - sudo dpkg-reconfigure --frontend --noninteractive tzdata + - sudo timedatectl set-timezone Asia/Harbin - cd ${TRAVIS_BUILD_DIR} - mkdir debug - cd debug @@ -203,8 +200,7 @@ matrix: - cmake before_script: - - echo 'Asia/Harbin' | sudo tee /etc/timezone - - sudo dpkg-reconfigure --frontend --noninteractive tzdata + - sudo timedatectl set-timezone Asia/Harbin - cd ${TRAVIS_BUILD_DIR} - mkdir debug - cd debug @@ -233,8 +229,7 @@ matrix: - DESC="trusty/gcc-4.8 build" before_script: - - echo 'Asia/Harbin' | sudo tee /etc/timezone - - sudo dpkg-reconfigure --frontend --noninteractive tzdata + - sudo timedatectl set-timezone Asia/Harbin - cd ${TRAVIS_BUILD_DIR} - mkdir debug - cd debug From ebf8d765d34050d2c06d605140d84b77fbdbee90 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Mon, 29 Jun 2020 19:34:24 +0800 Subject: [PATCH 38/64] [support maxSqlLen] --- src/kit/shell/src/shellEngine.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 044e167af7..0da01d9258 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -790,7 +790,7 @@ int isCommentLine(char *line) { void source_file(TAOS *con, char *fptr) { wordexp_t full_path; int read_len = 0; - char * cmd = calloc(1, MAX_COMMAND_SIZE); + char * cmd = calloc(1, tsMaxSQLStringLen+1); size_t cmd_len = 0; char * line = NULL; size_t line_len = 0; @@ -822,7 +822,7 @@ void source_file(TAOS *con, char *fptr) { } while ((read_len = getline(&line, &line_len, f)) != -1) { - if (read_len >= MAX_COMMAND_SIZE) continue; + if (read_len >= tsMaxSQLStringLen) continue; line[--read_len] = '\0'; if (read_len == 0 || isCommentLine(line)) { // line starts with # From 3588f00c8e3c206bf8f8dd2ba0071079319e7509 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 29 Jun 2020 19:39:37 +0800 Subject: [PATCH 39/64] show date --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0256df7907..95fc398486 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,6 +37,7 @@ matrix: before_script: - sudo timedatectl set-timezone Asia/Harbin + - date - cd ${TRAVIS_BUILD_DIR} - mkdir debug - cd debug @@ -152,6 +153,7 @@ matrix: before_script: - sudo timedatectl set-timezone Asia/Harbin + - date - cd ${TRAVIS_BUILD_DIR} - mkdir debug - cd debug @@ -176,6 +178,7 @@ matrix: before_script: - sudo timedatectl set-timezone Asia/Harbin + - date - cd ${TRAVIS_BUILD_DIR} - mkdir debug - cd debug @@ -201,6 +204,7 @@ matrix: before_script: - sudo timedatectl set-timezone Asia/Harbin + - date - cd ${TRAVIS_BUILD_DIR} - mkdir debug - cd debug @@ -230,6 +234,7 @@ matrix: before_script: - sudo timedatectl set-timezone Asia/Harbin + - date - cd ${TRAVIS_BUILD_DIR} - mkdir debug - cd debug From 700acfbe8526ee7a8d9f4ac1dc793587a6d878df Mon Sep 17 00:00:00 2001 From: Hui Li Date: Mon, 29 Jun 2020 19:40:06 +0800 Subject: [PATCH 40/64] [TD-794] --- src/kit/shell/src/shellEngine.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 0da01d9258..01dcc73ef6 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -839,7 +839,7 @@ void source_file(TAOS *con, char *fptr) { memcpy(cmd + cmd_len, line, read_len); printf("%s%s\n", PROMPT_HEADER, cmd); shellRunCommand(con, cmd); - memset(cmd, 0, MAX_COMMAND_SIZE); + memset(cmd, 0, tsMaxSQLStringLen); cmd_len = 0; } From 0d205a866320e477c037ec6d4a57eba7c26a6ccb Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 29 Jun 2020 20:05:19 +0800 Subject: [PATCH 41/64] another way to set timezone --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 95fc398486..2a0aa6372b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ matrix: - psmisc before_script: - - sudo timedatectl set-timezone Asia/Harbin + - export TZ=Asia/Harbin - date - cd ${TRAVIS_BUILD_DIR} - mkdir debug @@ -152,7 +152,7 @@ matrix: - DESC="trusty/gcc-4.8 build" before_script: - - sudo timedatectl set-timezone Asia/Harbin + - export TZ=Asia/Harbin - date - cd ${TRAVIS_BUILD_DIR} - mkdir debug @@ -177,7 +177,7 @@ matrix: - cmake before_script: - - sudo timedatectl set-timezone Asia/Harbin + - export TZ=Asia/Harbin - date - cd ${TRAVIS_BUILD_DIR} - mkdir debug @@ -203,7 +203,7 @@ matrix: - cmake before_script: - - sudo timedatectl set-timezone Asia/Harbin + - export TZ=Asia/Harbin - date - cd ${TRAVIS_BUILD_DIR} - mkdir debug @@ -233,7 +233,7 @@ matrix: - DESC="trusty/gcc-4.8 build" before_script: - - sudo timedatectl set-timezone Asia/Harbin + - export TZ=Asia/Harbin - date - cd ${TRAVIS_BUILD_DIR} - mkdir debug From 617d3519376e93c12443c60703d273249222253b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 29 Jun 2020 12:59:41 +0000 Subject: [PATCH 42/64] scripts --- tests/script/basicSuite.sim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index 0d82f96586..e498d517c7 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -6,10 +6,11 @@ run general/import/replica1.sim run general/parser/auto_create_tb_drop_tb.sim run general/parser/binary_escapeCharacter.sim run general/parser/select_from_cache_disk.sim +run general/parser/null_char.sim +run general/parser/alter.sim run general/parser/projection_limit_offset.sim run general/parser/join.sim run general/parser/where.sim -run general/parser/projection_limit_offset.sim run general/stable/vnode3.sim run general/tag/filter.sim run general/table/vgroup.sim From 357b7eabf90789a38b1bc9f1adb298d29cf4dbc3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 29 Jun 2020 13:17:35 +0000 Subject: [PATCH 43/64] for CI --- tests/script/basicSuite.sim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index e498d517c7..61a9d68d26 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -8,12 +8,13 @@ run general/parser/binary_escapeCharacter.sim run general/parser/select_from_cache_disk.sim run general/parser/null_char.sim run general/parser/alter.sim -run general/parser/projection_limit_offset.sim -run general/parser/join.sim -run general/parser/where.sim run general/stable/vnode3.sim run general/tag/filter.sim run general/table/vgroup.sim run general/user/authority.sim run general/vector/metrics_mix.sim run general/vector/table_field.sim +run general/user/authority.sim +run general/tag/set.sim +run general/table/delete_writing.sim +run general/stable/disk.sim From 13cc896cf5899b753e9bd1fec8612ea5f2c5f443 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 29 Jun 2020 13:41:31 +0000 Subject: [PATCH 44/64] rpc did not handle simeple request head in right way if its session is already closed in server coverity scan defect for tqueue.c --- src/common/src/tglobal.c | 2 +- src/rpc/src/rpcMain.c | 6 ++++++ src/rpc/src/rpcTcp.c | 2 +- src/rpc/src/rpcUdp.c | 2 +- src/util/src/tqueue.c | 7 ++++++- 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 6dc6a82d09..f70bcb936a 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -43,7 +43,7 @@ int16_t tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM; int32_t tsNumOfMnodes = 3; // common -int32_t tsRpcTimer = 300; +int32_t tsRpcTimer = 1000; int32_t tsRpcMaxTime = 600; // seconds; int32_t tsMaxShellConns = 5000; int32_t tsMaxConnections = 5000; diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 3ab0dddb5c..ad1e971453 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -665,6 +665,12 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { return pConn; } + // if code is not 0, it means it is simple reqhead, just ignore + if (pHead->code != 0) { + terrno = TSDB_CODE_RPC_ALREADY_PROCESSED; + return NULL; + } + int sid = taosAllocateId(pRpc->idPool); if (sid <= 0) { tError("%s maximum number of sessions:%d is reached", pRpc->label, pRpc->sessions); diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index ff8d055fd4..def1fd9383 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -419,7 +419,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); return -1; } else { - // tDebug("malloc mem: %p", buffer); + tDebug("TCP malloc mem: %p", buffer); } msg = buffer + tsRpcOverhead; diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 91324da474..decd7a8307 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -212,7 +212,7 @@ static void *taosRecvUdpData(void *param) { tError("%s failed to allocate memory, size:%ld", pConn->label, dataLen); continue; } else { - // tTrace("malloc mem: %p", tmsg); + tDebug("UDP malloc mem: %p", tmsg); } tmsg += tsRpcOverhead; // overhead for SRpcReqContext diff --git a/src/util/src/tqueue.c b/src/util/src/tqueue.c index 34396c9c29..b499f34060 100644 --- a/src/util/src/tqueue.c +++ b/src/util/src/tqueue.c @@ -68,10 +68,15 @@ void taosCloseQueue(taos_queue param) { if (param == NULL) return; STaosQueue *queue = (STaosQueue *)param; STaosQnode *pTemp; + STaosQset *qset; + + pthread_mutex_lock(&queue->mutex); STaosQnode *pNode = queue->head; queue->head = NULL; + qset = queue->qset; + pthread_mutex_unlock(&queue->mutex); - if (queue->qset) taosRemoveFromQset(queue->qset, queue); + if (queue->qset) taosRemoveFromQset(qset, queue); pthread_mutex_lock(&queue->mutex); From 1f9ebc0d48dbdf8ce542e3876fb9f1f1dbdf68d0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 29 Jun 2020 23:19:35 +0800 Subject: [PATCH 45/64] [td-225] opt query perf --- src/query/src/qExecutor.c | 40 +++++++++------------------------------ src/query/src/qfill.c | 3 +-- src/tsdb/src/tsdbRead.c | 25 ++++++++++-------------- src/vnode/src/vnodeRead.c | 1 - 4 files changed, 20 insertions(+), 49 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index cbc7d23c0a..8a304ab810 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3291,16 +3291,17 @@ void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols) free(pTableQueryInfo); } -void setCurrentQueryTable(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo) { - SQuery *pQuery = pRuntimeEnv->pQuery; - pQuery->current = pTableQueryInfo; - - assert(((pTableQueryInfo->lastKey >= pTableQueryInfo->win.skey) && QUERY_IS_ASC_QUERY(pQuery)) || - ((pTableQueryInfo->lastKey <= pTableQueryInfo->win.skey) && !QUERY_IS_ASC_QUERY(pQuery))); -} +#define SET_CURRENT_QUERY_TABLE_INFO(_runtime, _tableInfo) \ + do { \ + SQuery *_query = (_runtime)->pQuery; \ + _query->current = _tableInfo; \ + assert((((_tableInfo)->lastKey >= (_tableInfo)->win.skey) && QUERY_IS_ASC_QUERY(_query)) || \ + (((_tableInfo)->lastKey <= (_tableInfo)->win.skey) && !QUERY_IS_ASC_QUERY(_query))); \ + } while (0) /** * set output buffer for different group + * TODO opt performance if current group is identical to previous group * @param pRuntimeEnv * @param pDataBlockInfo */ @@ -4165,32 +4166,9 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { if(pTableQueryInfo == NULL) { break; } - // todo opt performance using hash table - -// size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo); -// for (int32_t i = 0; i < numOfGroup; ++i) { -// SArray *group = GET_TABLEGROUP(pQInfo, i); -// -// size_t num = taosArrayGetSize(group); -// for (int32_t j = 0; j < num; ++j) { -// STableQueryInfo *p = taosArrayGetP(group, j); -// -// STableId id = tsdbGetTableId(p->pTable); -// if (id.tid == blockInfo.tid) { -// assert(id.uid == blockInfo.uid); -// pTableQueryInfo = p; -// -// break; -// } -// } -// -// if (pTableQueryInfo != NULL) { -// break; -// } -// } assert(*pTableQueryInfo != NULL); - setCurrentQueryTable(pRuntimeEnv, *pTableQueryInfo); + SET_CURRENT_QUERY_TABLE_INFO(pRuntimeEnv, *pTableQueryInfo); SDataStatis *pStatis = NULL; SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis); diff --git a/src/query/src/qfill.c b/src/query/src/qfill.c index 59bf7b423c..4cb3779166 100644 --- a/src/query/src/qfill.c +++ b/src/query/src/qfill.c @@ -209,8 +209,7 @@ int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) { return 0; } - return FILL_IS_ASC_FILL(pFillInfo) ? (pFillInfo->numOfRows - pFillInfo->rowIdx) - : pFillInfo->rowIdx + 1; + return FILL_IS_ASC_FILL(pFillInfo) ? (pFillInfo->numOfRows - pFillInfo->rowIdx) : pFillInfo->rowIdx + 1; } // todo: refactor diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 41217f0f4f..19a022e0a7 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -542,17 +542,12 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo return TSDB_CODE_SUCCESS; } -static SDataBlockInfo getTrueDataBlockInfo(STableCheckInfo* pCheckInfo, SCompBlock* pBlock) { - SDataBlockInfo info = { - .window = {.skey = pBlock->keyFirst, .ekey = pBlock->keyLast}, - .numOfCols = pBlock->numOfCols, - .rows = pBlock->numOfRows, - .tid = pCheckInfo->tableId.tid, - .uid = pCheckInfo->tableId.uid, - }; - - return info; -} +#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ + ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ + .numOfCols = (_block)->numOfCols, \ + .rows = (_block)->numOfRows, \ + .tid = (_checkInfo)->tableId.tid, \ + .uid = (_checkInfo)->tableId.uid}) static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) { size_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); @@ -626,7 +621,7 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){ SQueryFilePos* cur = &pQueryHandle->cur; - SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlock); + SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo); SDataRow row = getSDataRowInTableMem(pCheckInfo); @@ -946,7 +941,7 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock, SArray* sa) { SQueryFilePos* cur = &pQueryHandle->cur; - SDataBlockInfo blockInfo = getTrueDataBlockInfo(pCheckInfo, pBlock); + SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); initTableMemIterator(pQueryHandle, pCheckInfo); SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; @@ -1322,8 +1317,8 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables); // the pTableQueryInfo[j]->numOfBlocks may be 0 sup.numOfTables = numOfQualTables; - SLoserTreeInfo* pTree = NULL; + SLoserTreeInfo* pTree = NULL; uint8_t ret = tLoserTreeCreate(&pTree, sup.numOfTables, &sup, dataBlockOrderCompar); if (ret != TSDB_CODE_SUCCESS) { cleanBlockOrderSupporter(&sup, numOfTables); @@ -1844,7 +1839,7 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { if (pHandle->cur.mixBlock) { return pHandle->pColumns; } else { - SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlockInfo->compBlock); + SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock); assert(pHandle->realNumOfRows <= binfo.rows); // data block has been loaded, todo extract method diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 894d214d85..0c08c77e32 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -137,7 +137,6 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } if (pQInfo != NULL) { - vDebug("vgId:%d, QInfo:%p, do qTableQuery", pVnode->vgId, pQInfo); qTableQuery(pQInfo, vnodeRelease, pVnode); // do execute query } From 607959a0825c48d96072d3c8c95f6500ab8bf662 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 29 Jun 2020 15:32:35 +0000 Subject: [PATCH 46/64] check the application message length before passing it to server --- src/rpc/src/rpcMain.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index ad1e971453..a59024f8db 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1034,15 +1034,20 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { rpcMsg.ahandle = pConn->ahandle; if ( rpcIsReq(pHead->msgType) ) { - rpcMsg.handle = pConn; - rpcAddRef(pRpc); // add the refCount for requests + if (rpcMsg.contLen > 0) { + rpcMsg.handle = pConn; + rpcAddRef(pRpc); // add the refCount for requests - // start the progress timer to monitor the response from server app - if (pConn->connType != RPC_CONN_TCPS) - pConn->pTimer = taosTmrStart(rpcProcessProgressTimer, tsProgressTimer, pConn, pRpc->tmrCtrl); + // start the progress timer to monitor the response from server app + if (pConn->connType != RPC_CONN_TCPS) + pConn->pTimer = taosTmrStart(rpcProcessProgressTimer, tsProgressTimer, pConn, pRpc->tmrCtrl); - // notify the server app - (*(pRpc->cfp))(&rpcMsg, NULL); + // notify the server app + (*(pRpc->cfp))(&rpcMsg, NULL); + } else { + tDebug("%s, message body is empty, ignore", pConn->info); + rpcFreeCont(rpcMsg.pCont); + } } else { // it's a response SRpcReqContext *pContext = pConn->pContext; From 28219a84c1182712dbe07da25792b3c48f91afc2 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Tue, 30 Jun 2020 09:23:21 +0800 Subject: [PATCH 47/64] fix td-779 --- src/mnode/src/mnodeTable.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 2d9892cf01..7dc0c5dfa5 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -2374,6 +2374,17 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows strcat(prefix, TS_PATH_DELIMITER); int32_t prefixLen = strlen(prefix); + char* pattern = NULL; + if (pShow->payloadLen > 0) { + pattern = (char*)malloc(pShow->payloadLen + 1); + if (pattern == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return 0; + } + memcpy(pattern, pShow->payload, pShow->payloadLen); + pattern[pShow->payloadLen] = 0; + } + while (numOfRows < rows) { pShow->pIter = mnodeGetNextChildTable(pShow->pIter, &pTable); if (pTable == NULL) break; @@ -2389,7 +2400,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows // pattern compare for table name mnodeExtractTableName(pTable->info.tableId, tableName); - if (pShow->payloadLen > 0 && patternMatch(pShow->payload, tableName, sizeof(tableName) - 1, &info) != TSDB_PATTERN_MATCH) { + if (pattern != NULL && patternMatch(pattern, tableName, sizeof(tableName) - 1, &info) != TSDB_PATTERN_MATCH) { mnodeDecTableRef(pTable); continue; } @@ -2433,6 +2444,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows mnodeVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow); mnodeDecDbRef(pDb); + free(pattern); return numOfRows; } From c024afe67f64f45e1e682563c93a36736a3c6c79 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 30 Jun 2020 09:34:30 +0800 Subject: [PATCH 48/64] [TD-795] --- src/inc/taosmsg.h | 2 +- src/mnode/inc/mnodeDef.h | 2 +- src/mnode/src/mnodeVgroup.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 78255a45d5..c8bd2c6076 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -618,7 +618,7 @@ typedef struct { } SMDVnodeDesc; typedef struct { - char db[TSDB_DB_NAME_LEN]; + char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; SMDVnodeCfg cfg; SMDVnodeDesc nodes[TSDB_MAX_REPLICA]; } SMDCreateVnodeMsg; diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index ae1ba98a0f..46d2675705 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -132,7 +132,7 @@ typedef struct SVgObj { int64_t createdTime; int32_t lbDnodeId; int32_t lbTime; - char dbName[TSDB_DB_NAME_LEN]; + char dbName[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; int8_t inUse; int8_t accessState; int8_t reserved0[5]; diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 36fa306c9f..b314bfc8d0 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -358,7 +358,7 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg, SDbObj *pDb) { if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj)); - tstrncpy(pVgroup->dbName, pDb->name, TSDB_DB_NAME_LEN); + tstrncpy(pVgroup->dbName, pDb->name, TSDB_ACCT_LEN + TSDB_DB_NAME_LEN); pVgroup->numOfVnodes = pDb->cfg.replications; pVgroup->createdTime = taosGetTimestampMs(); pVgroup->accessState = TSDB_VN_ALL_ACCCESS; From ee4e4aaa9cb76283968d156824ff785af0e2c97c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 30 Jun 2020 02:31:23 +0000 Subject: [PATCH 49/64] change some logs from debug to trace --- src/mnode/src/mnodeSdb.c | 1 + src/tsdb/src/tsdbMemTable.c | 2 +- src/tsdb/src/tsdbMeta.c | 4 ++-- src/util/src/tqueue.c | 8 ++++---- src/vnode/src/vnodeWrite.c | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index e1d255ee76..761dce6720 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -367,6 +367,7 @@ void sdbCleanUp() { tsSdbObj.status = SDB_STATUS_CLOSING; sdbCleanupWriteWorker(); + sdbDebug("sdb will be closed, version:%" PRId64, tsSdbObj.version); if (tsSdbObj.sync) { syncStop(tsSdbObj.sync); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 12db79182e..d3b9081a36 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -111,7 +111,7 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { ASSERT(pTableData->numOfRows == tSkipListGetSize(pTableData->pData)); } - tsdbDebug("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), + tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), key); return 0; diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index b769a87beb..3b25d68896 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -596,7 +596,7 @@ static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { return -1; } - tsdbDebug("vgId:%d table %s tid %d uid %" PRIu64 " is restored from file", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), + tsdbTrace("vgId:%d table %s tid %d uid %" PRIu64 " is restored from file", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); return 0; } @@ -797,7 +797,7 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx) { pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), pTable->sql, tsdbGetTableSchema(pTable)); } - tsdbDebug("vgId:%d table %s tid %d uid %" PRIu64 " is added to meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), + tsdbTrace("vgId:%d table %s tid %d uid %" PRIu64 " is added to meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); return 0; diff --git a/src/util/src/tqueue.c b/src/util/src/tqueue.c index 34396c9c29..0dabcdcff7 100644 --- a/src/util/src/tqueue.c +++ b/src/util/src/tqueue.c @@ -95,7 +95,7 @@ void *taosAllocateQitem(int size) { void taosFreeQitem(void *param) { if (param == NULL) return; - uDebug("item:%p is freed", param); + uTrace("item:%p is freed", param); char *temp = (char *)param; temp -= sizeof(STaosQnode); free(temp); @@ -119,7 +119,7 @@ int taosWriteQitem(taos_queue param, int type, void *item) { queue->numOfItems++; if (queue->qset) atomic_add_fetch_32(&queue->qset->numOfItems, 1); - uDebug("item:%p is put into queue:%p, type:%d items:%d", item, queue, type, queue->numOfItems); + uTrace("item:%p is put into queue:%p, type:%d items:%d", item, queue, type, queue->numOfItems); pthread_mutex_unlock(&queue->mutex); @@ -201,7 +201,7 @@ int taosGetQitem(taos_qall param, int *type, void **pitem) { *pitem = pNode->item; *type = pNode->type; num = 1; - uDebug("item:%p is fetched, type:%d", *pitem, *type); + uTrace("item:%p is fetched, type:%d", *pitem, *type); } return num; @@ -339,7 +339,7 @@ int taosReadQitemFromQset(taos_qset param, int *type, void **pitem, void **phand queue->numOfItems--; atomic_sub_fetch_32(&qset->numOfItems, 1); code = 1; - uDebug("item:%p is read out from queue:%p, type:%d items:%d", *pitem, queue, *type, queue->numOfItems); + uTrace("item:%p is read out from queue:%p, type:%d items:%d", *pitem, queue, *type, queue->numOfItems); } pthread_mutex_unlock(&queue->mutex); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index f43d10a7a3..171557acb6 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -94,7 +94,7 @@ static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pR // save insert result into item - vDebug("vgId:%d, submit msg is processed", pVnode->vgId); + vTrace("vgId:%d, submit msg is processed", pVnode->vgId); pRet->len = sizeof(SShellSubmitRspMsg); pRet->rsp = rpcMallocCont(pRet->len); From d2948b9170cf8b1e0f733fe9214ab1da23e1a88c Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 30 Jun 2020 10:56:32 +0800 Subject: [PATCH 50/64] add two more cases to regression. --- tests/script/regressionSuite.sim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/script/regressionSuite.sim b/tests/script/regressionSuite.sim index c671cf47c6..f4ffb5ea9e 100644 --- a/tests/script/regressionSuite.sim +++ b/tests/script/regressionSuite.sim @@ -142,6 +142,8 @@ run general/stable/dnode3.sim run general/stable/metrics.sim run general/stable/values.sim run general/stable/vnode3.sim +run general/stable/refcount.sim +run general/stable/show.sim run general/table/autocreate.sim run general/table/basic1.sim run general/table/basic2.sim From c64f9e73a2eb3f8b1126bd838e8a663700410daa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 30 Jun 2020 11:21:22 +0800 Subject: [PATCH 51/64] [td-225] opt query perf --- src/query/inc/qExecutor.h | 7 ++++--- src/query/src/qExecutor.c | 22 ++++++++++++++-------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index ee61f4f702..a830723845 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -161,12 +161,12 @@ typedef struct SQuery { } SQuery; typedef struct SQueryRuntimeEnv { - SResultInfo* resultInfo; // todo refactor to merge with SWindowResInfo + SResultInfo* resultInfo; // todo refactor to merge with SWindowResInfo SQuery* pQuery; SQLFunctionCtx* pCtx; int16_t numOfRowsPerPage; int16_t offset[TSDB_MAX_COLUMNS]; - uint16_t scanFlag; // denotes reversed scan of data or not + uint16_t scanFlag; // denotes reversed scan of data or not SFillInfo* pFillInfo; SWindowResInfo windowResInfo; STSBuf* pTSBuf; @@ -176,7 +176,8 @@ typedef struct SQueryRuntimeEnv { void* pQueryHandle; void* pSecQueryHandle; // another thread for SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file - bool topBotQuery; // false; + bool topBotQuery; // false + int32_t prevGroupId; // previous executed group id } SQueryRuntimeEnv; typedef struct SQInfo { diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 8a304ab810..35880232f9 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3305,13 +3305,16 @@ void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols) * @param pRuntimeEnv * @param pDataBlockInfo */ -void setExecutionContext(SQInfo *pQInfo, void* pTable, int32_t groupIndex, TSKEY nextKey) { +void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - STableQueryInfo *pTableQueryInfo = pRuntimeEnv->pQuery->current; - - SWindowResInfo * pWindowResInfo = &pRuntimeEnv->windowResInfo; - int32_t GROUPRESULTID = 1; + STableQueryInfo *pTableQueryInfo = pRuntimeEnv->pQuery->current; + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + if (pRuntimeEnv->prevGroupId != INT32_MIN && pRuntimeEnv->prevGroupId == groupIndex) { + return; + } + + int32_t GROUPRESULTID = 1; SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, sizeof(groupIndex)); if (pWindowRes == NULL) { return; @@ -3328,6 +3331,8 @@ void setExecutionContext(SQInfo *pQInfo, void* pTable, int32_t groupIndex, TSKEY } } + // record the current active group id + pRuntimeEnv->prevGroupId = groupIndex; setWindowResOutputBuf(pRuntimeEnv, pWindowRes); initCtxOutputBuf(pRuntimeEnv); @@ -4072,6 +4077,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool pRuntimeEnv->pTSBuf = param; pRuntimeEnv->cur.vgroupIndex = -1; pRuntimeEnv->stableQuery = isSTableQuery; + pRuntimeEnv->prevGroupId = INT32_MIN; if (param != NULL) { int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; @@ -4176,8 +4182,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { if (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) { if (!isIntervalQuery(pQuery)) { int32_t step = QUERY_IS_ASC_QUERY(pQuery)? 1:-1; - setExecutionContext(pQInfo, (*pTableQueryInfo)->pTable, (*pTableQueryInfo)->groupIndex, - blockInfo.window.ekey + step); + setExecutionContext(pQInfo, (*pTableQueryInfo)->groupIndex, blockInfo.window.ekey + step); } else { // interval query TSKEY nextKey = blockInfo.window.skey; setIntervalQueryRange(pQInfo, nextKey); @@ -4553,7 +4558,8 @@ static void doSaveContext(SQInfo *pQInfo) { if (pRuntimeEnv->pSecQueryHandle != NULL) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - + + pRuntimeEnv->prevGroupId = INT32_MIN; pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); setQueryStatus(pQuery, QUERY_NOT_COMPLETED); From 555206e9b1bc01718be284ee171cf32b9d8ab828 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 30 Jun 2020 04:35:21 +0000 Subject: [PATCH 52/64] [TD-800] crash while connect failed --- src/mnode/src/mnodeShow.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 6f5216dfa3..06ef2cb452 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -281,6 +281,7 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { SCMConnectMsg *pConnectMsg = pMsg->rpcMsg.pCont; + SCMConnectRsp *pConnectRsp = NULL; int32_t code = TSDB_CODE_SUCCESS; SRpcConnInfo connInfo; @@ -309,7 +310,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { mnodeDecDbRef(pDb); } - SCMConnectRsp *pConnectRsp = rpcMallocCont(sizeof(SCMConnectRsp)); + pConnectRsp = rpcMallocCont(sizeof(SCMConnectRsp)); if (pConnectRsp == NULL) { code = TSDB_CODE_MND_OUT_OF_MEMORY; goto connect_over; @@ -332,7 +333,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { connect_over: if (code != TSDB_CODE_SUCCESS) { - rpcFreeCont(pConnectRsp); + if (pConnectRsp) rpcFreeCont(pConnectRsp); mLError("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code)); } else { mLInfo("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code)); From cdee839a776d0f764b8f51937300ba8b0d372d98 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 30 Jun 2020 07:03:24 +0000 Subject: [PATCH 53/64] ref count error --- src/mnode/src/mnodeTable.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 7dc0c5dfa5..65900a60db 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -457,10 +457,9 @@ static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) { free(pNew); free(oldTableId); free(oldSchema); - - mnodeDecTableRef(pTable); } + mnodeDecTableRef(pTable); return TSDB_CODE_SUCCESS; } From 92d81c703c5dc52e48cd6f654ada6a515bf5933b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 30 Jun 2020 15:23:50 +0800 Subject: [PATCH 54/64] [td-808] build submit block along with table schema. --- src/client/inc/tscLocalMerge.h | 2 +- src/client/inc/tscSubquery.h | 2 + src/client/inc/tsclient.h | 14 +++-- src/client/src/tscAsync.c | 93 ++++++++++++++++++++++------------ src/client/src/tscServer.c | 8 +-- src/client/src/tscSubquery.c | 42 ++++++++++++--- src/client/src/tscUtil.c | 50 ++++++++++++------ src/inc/taosmsg.h | 3 +- src/tsdb/src/tsdbMain.c | 13 ++--- 9 files changed, 156 insertions(+), 71 deletions(-) diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index c073f40546..d38885ab2e 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -84,7 +84,7 @@ typedef struct SRetrieveSupport { SColumnModel * pFinalColModel; // colModel for final result SSubqueryState * pState; int32_t subqueryIndex; // index of current vnode in vnode list - SSqlObj * pParentSqlObj; + SSqlObj * pParentSql; tFilePage * localBuffer; // temp buffer, there is a buffer for each vnode to uint32_t numOfRetry; // record the number of retry times pthread_mutex_t queryMutex; diff --git a/src/client/inc/tscSubquery.h b/src/client/inc/tscSubquery.h index 82d490376a..d5833675aa 100644 --- a/src/client/inc/tscSubquery.h +++ b/src/client/inc/tscSubquery.h @@ -36,6 +36,8 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql); int32_t tscHandleMultivnodeInsert(SSqlObj *pSql); +int32_t tscHandleInsertRetry(SSqlObj* pSql); + void tscBuildResFromSubqueries(SSqlObj *pSql); void **doSetResultRowData(SSqlObj *pSql, bool finalResult); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 471db9c99f..6ce94d5aa4 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -213,8 +213,7 @@ typedef struct SQueryInfo { typedef struct { int command; uint8_t msgType; - bool autoCreated; // if the table is missing, on-the-fly create it. during getmeterMeta - int8_t dataSourceType; // load data from file or not + bool autoCreated; // create table if it is not existed during retrieve table meta in mnode union { int32_t count; @@ -222,18 +221,23 @@ typedef struct { }; int32_t insertType; - int32_t clauseIndex; // index of multiple subclause query + int32_t clauseIndex; // index of multiple subclause query + + char * curSql; // current sql, resume position of sql after parsing paused int8_t parseFinished; + short numOfCols; uint32_t allocSize; char * payload; int32_t payloadLen; SQueryInfo **pQueryInfo; int32_t numOfClause; - char * curSql; // current sql, resume position of sql after parsing paused - void * pTableList; // referred table involved in sql int32_t batchSize; // for parameter ('?') binding and batch processing int32_t numOfParams; + + int8_t dataSourceType; // load data from file or not + int8_t submitSchema; // submit block is built with table schema + SHashObj *pTableList; // referred table involved in sql SArray *pDataBlocks; // SArray submit data blocks after parsing sql } SSqlCmd; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 0bfee46b4b..2de45bcc6e 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -431,6 +431,8 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { return; } + tscDebug("%p get tableMeta successfully", pSql); + if (pSql->pStream == NULL) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -446,20 +448,20 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { assert(code == TSDB_CODE_SUCCESS); } - - assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pTableMetaInfo->vgroupIndex >= 0 && pSql->param != NULL); + assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pSql->param != NULL); SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param; - SSqlObj * pParObj = trs->pParentSqlObj; + SSqlObj * pParObj = trs->pParentSql; - assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex && - tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0); - // NOTE: the vgroupInfo for the queried super table must be existed here. - assert(pTableMetaInfo->vgroupList != NULL); + assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex && + pTableMetaInfo->vgroupIndex >= 0 && pTableMetaInfo->vgroupList != NULL); + if ((code = tscProcessSql(pSql)) == TSDB_CODE_SUCCESS) { return; } + + goto _error; } else { // continue to process normal async query if (pCmd->parseFinished) { tscDebug("%p update table meta in local cache, continue to process sql and send corresponding query", pSql); @@ -472,18 +474,41 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { assert(code == TSDB_CODE_SUCCESS); } - // if failed to process sql, go to error handler - if ((code = tscProcessSql(pSql)) == TSDB_CODE_SUCCESS) { - return; + // in case of insert, redo parsing the sql string and build new submit data block for two reasons: + // 1. the table Id(tid & uid) may have been update, the submit block needs to be updated + // 2. vnode may need the schema information along with submit block to update its local table schema. + if (pCmd->command == TSDB_SQL_INSERT) { + tscDebug("%p redo parse sql string to build submit block", pSql); + + pCmd->parseFinished = false; + if ((code = tsParseSql(pSql, true)) == TSDB_CODE_SUCCESS) { + /* + * Discard previous built submit blocks, and then parse the sql string again and build up all submit blocks, + * and send the required submit block according to index value in supporter to server. + */ + pSql->fp = pSql->fetchFp; // restore the fp + if ((code = tscHandleInsertRetry(pSql)) == TSDB_CODE_SUCCESS) { + return; + } + } + + } else {// in case of other query type, continue + if ((code = tscProcessSql(pSql)) == TSDB_CODE_SUCCESS) { + return; + } } -// // todo update the submit message according to the new table meta -// // 1. table uid, 2. ip address -// code = tscSendMsgToServer(pSql); -// if (code == TSDB_CODE_SUCCESS) return; + + goto _error; } else { tscDebug("%p continue parse sql after get table meta", pSql); code = tsParseSql(pSql, false); + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + return; + } else if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STMT_INSERT)) { STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); code = tscGetTableMeta(pSql, pTableMetaInfo); @@ -492,45 +517,49 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { } else { assert(code == TSDB_CODE_SUCCESS); } + (*pSql->fp)(pSql->param, pSql, code); return; } - - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return; + + // proceed to invoke the tscDoQuery(); } } } else { // stream computing STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - code = tscGetTableMeta(pSql, pTableMetaInfo); - pRes->code = code; - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return; + code = tscGetTableMeta(pSql, pTableMetaInfo); + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + return; + } else if (code != TSDB_CODE_SUCCESS) { + goto _error; + } if (code == TSDB_CODE_SUCCESS && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { code = tscGetSTableVgroupInfo(pSql, pCmd->clauseIndex); - pRes->code = code; - - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return; + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + return; + } else if (code != TSDB_CODE_SUCCESS) { + goto _error; + } } - } - if (code != TSDB_CODE_SUCCESS) { - pSql->res.code = code; - tscQueueAsyncRes(pSql); - return; - } - - if (pSql->pStream) { tscDebug("%p stream:%p meta is updated, start new query, command:%d", pSql, pSql->pStream, pSql->cmd.command); if (!pSql->cmd.parseFinished) { tsParseSql(pSql, false); sem_post(&pSql->rspSem); } + return; - } else { - tscDebug("%p get tableMeta successfully", pSql); } tscDoQuery(pSql); + return; + + _error: + if (code != TSDB_CODE_SUCCESS) { + pSql->res.code = code; + tscQueueAsyncRes(pSql); + } } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 470a829805..8dd5ef69f7 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -347,8 +347,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { int doProcessSql(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - int32_t code = TSDB_CODE_SUCCESS; - + if (pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || pCmd->command == TSDB_SQL_RETRIEVE || @@ -365,10 +364,13 @@ int doProcessSql(SSqlObj *pSql) { return pRes->code; } - code = tscSendMsgToServer(pSql); + int32_t code = tscSendMsgToServer(pSql); + + // NOTE: if code is TSDB_CODE_SUCCESS, pSql may have been released here already by other threads. if (code != TSDB_CODE_SUCCESS) { pRes->code = code; tscQueueAsyncRes(pSql); + return pRes->code; } return TSDB_CODE_SUCCESS; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 0c2e3f3a1c..b9ba22436e 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -25,6 +25,7 @@ typedef struct SInsertSupporter { SSubqueryState* pState; SSqlObj* pSql; + int32_t index; } SInsertSupporter; static void freeJoinSubqueryObj(SSqlObj* pSql); @@ -1876,13 +1877,36 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) // release data block data tfree(pState); -// pParentCmd->pDataBlocks = tscDestroyBlockArrayList(pParentCmd->pDataBlocks); - + // restore user defined fp pParentObj->fp = pParentObj->fetchFp; - + + // todo remove this parameter in async callback function definition. // all data has been sent to vnode, call user function - (*pParentObj->fp)(pParentObj->param, pParentObj, numOfRows); + int32_t v = (pParentObj->res.code != TSDB_CODE_SUCCESS)? pParentObj->res.code:pParentObj->res.numOfRows; + (*pParentObj->fp)(pParentObj->param, pParentObj, v); +} + +/** + * it is a subquery, so after parse the sql string, copy the submit block to payload of itself + * @param pSql + * @return + */ +int32_t tscHandleInsertRetry(SSqlObj* pSql) { + assert(pSql != NULL && pSql->param != NULL); + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + + SInsertSupporter* pSupporter = (SInsertSupporter*) pSql->param; + assert(pSupporter->index < pSupporter->pState->numOfTotal); + + STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->pDataBlocks, pSupporter->index); + pRes->code = tscCopyDataBlockToPayload(pSql, pTableDataBlock); + if (pRes->code != TSDB_CODE_SUCCESS) { + return pRes->code; + } + + return tscProcessSql(pSql); } int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { @@ -1906,10 +1930,11 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { while(numOfSub < pSql->numOfSubs) { SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter)); - pSupporter->pSql = pSql; + pSupporter->pSql = pSql; pSupporter->pState = pState; - - SSqlObj *pNew = createSimpleSubObj(pSql, multiVnodeInsertFinalize, pSupporter, TSDB_SQL_INSERT);//createSubqueryObj(pSql, 0, multiVnodeInsertFinalize, pSupporter1, TSDB_SQL_INSERT, NULL); + pSupporter->index = numOfSub; + + SSqlObj *pNew = createSimpleSubObj(pSql, multiVnodeInsertFinalize, pSupporter, TSDB_SQL_INSERT); if (pNew == NULL) { tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, numOfSub, strerror(errno)); goto _error; @@ -1940,6 +1965,8 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { return pRes->code; // free all allocated resource } + pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); + // use the local variable for (int32_t j = 0; j < numOfSub; ++j) { SSqlObj *pSub = pSql->pSubs[j]; @@ -1947,7 +1974,6 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { tscProcessSql(pSub); } - pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); return TSDB_CODE_SUCCESS; _error: diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index d15a0d7fcc..259bcd4cbd 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -562,10 +562,8 @@ int32_t tscGetDataBlockFromList(void* pHashList, SArray* pDataBlockList, int64_t return TSDB_CODE_SUCCESS; } -static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) { +static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bool includeSchema) { // TODO: optimize this function, handle the case while binary is not presented - int len = 0; - STableMeta* pTableMeta = pTableDataBlock->pTableMeta; STableComInfo tinfo = tscGetTableInfo(pTableMeta); SSchema* pSchema = tscGetTableSchema(pTableMeta); @@ -575,16 +573,37 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) { pDataBlock += sizeof(SSubmitBlk); int32_t flen = 0; // original total length of row - for (int32_t i = 0; i < tinfo.numOfColumns; ++i) { - flen += TYPE_BYTES[pSchema[i].type]; + + // schema needs to be included into the submit data block + if (includeSchema) { + int32_t numOfCols = tscGetNumOfColumns(pTableDataBlock->pTableMeta); + for(int32_t j = 0; j < numOfCols; ++j) { + STColumn* pCol = (STColumn*) pDataBlock; + pCol->colId = pSchema[j].colId; + pCol->type = pSchema[j].type; + pCol->bytes = pSchema[j].bytes; + pCol->offset = 0; + + pDataBlock += sizeof(STColumn); + flen += TYPE_BYTES[pSchema[j].type]; + } + + int32_t schemaSize = sizeof(STColumn) * numOfCols; + pBlock->schemaLen = schemaSize; + } else { + for (int32_t j = 0; j < tinfo.numOfColumns; ++j) { + flen += TYPE_BYTES[pSchema[j].type]; + } + + pBlock->schemaLen = 0; } char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); - pBlock->len = 0; + pBlock->dataLen = 0; int32_t numOfRows = htons(pBlock->numOfRows); for (int32_t i = 0; i < numOfRows; ++i) { - SDataRow trow = (SDataRow)pDataBlock; + SDataRow trow = (SDataRow) pDataBlock; dataRowSetLen(trow, TD_DATA_ROW_HEAD_SIZE + flen); dataRowSetVersion(trow, pTableMeta->sversion); @@ -595,20 +614,21 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) { p += pSchema[j].bytes; } - // p += pTableDataBlock->rowSize; pDataBlock += dataRowLen(trow); - pBlock->len += dataRowLen(trow); + pBlock->dataLen += dataRowLen(trow); } - len = pBlock->len; - pBlock->len = htonl(pBlock->len); + int32_t len = pBlock->dataLen + pBlock->schemaLen; + pBlock->dataLen = htonl(pBlock->dataLen); + pBlock->schemaLen = htonl(pBlock->schemaLen); + return len; } int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { SSqlCmd* pCmd = &pSql->cmd; - // the expanded size when a row data is converted to SDataRow format + // the maximum expanded size in byte when a row-wise data is converted to SDataRow format const int32_t MAX_EXPAND_SIZE = TD_DATA_ROW_HEAD_SIZE + TYPE_BYTES[TSDB_DATA_TYPE_BINARY]; void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); @@ -617,7 +637,6 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { size_t total = taosArrayGetSize(pTableDataBlockList); for (int32_t i = 0; i < total; ++i) { STableDataBlocks* pOneTableBlock = taosArrayGetP(pTableDataBlockList, i); - STableDataBlocks* dataBuf = NULL; int32_t ret = @@ -666,16 +685,17 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { pBlocks->uid = htobe64(pBlocks->uid); pBlocks->sversion = htonl(pBlocks->sversion); pBlocks->numOfRows = htons(pBlocks->numOfRows); + pBlocks->schemaLen = 0; // erase the empty space reserved for binary data - int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock); + int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, pCmd->submitSchema); assert(finalLen <= len); dataBuf->size += (finalLen + sizeof(SSubmitBlk)); assert(dataBuf->size <= dataBuf->nAllocSize); // the length does not include the SSubmitBlk structure - pBlocks->len = htonl(finalLen); + pBlocks->dataLen = htonl(finalLen); dataBuf->numOfTables += 1; } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 78255a45d5..d4a73e0e33 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -192,7 +192,8 @@ typedef struct SSubmitBlk { int32_t tid; // table id int32_t padding; // TODO just for padding here int32_t sversion; // data schema version - int32_t len; // data part length, not including the SSubmitBlk head + int32_t dataLen; // data part length, not including the SSubmitBlk head + int32_t schemaLen; // schema length, if length is 0, no schema exists int16_t numOfRows; // total number of rows in current submit block char data[]; } SSubmitBlk; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 6c6b313073..e650cef45c 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -768,7 +768,8 @@ static SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter) { SSubmitBlk *pBlock = pIter->pBlock; if (pBlock == NULL) return NULL; - pBlock->len = htonl(pBlock->len); + pBlock->dataLen = htonl(pBlock->dataLen); + pBlock->schemaLen = htonl(pBlock->schemaLen); pBlock->numOfRows = htons(pBlock->numOfRows); pBlock->uid = htobe64(pBlock->uid); pBlock->tid = htonl(pBlock->tid); @@ -776,11 +777,11 @@ static SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter) { pBlock->sversion = htonl(pBlock->sversion); pBlock->padding = htonl(pBlock->padding); - pIter->len = pIter->len + sizeof(SSubmitBlk) + pBlock->len; + pIter->len = pIter->len + sizeof(SSubmitBlk) + pBlock->dataLen; if (pIter->len >= pIter->totalLen) { pIter->pBlock = NULL; } else { - pIter->pBlock = (SSubmitBlk *)((char *)pBlock + pBlock->len + sizeof(SSubmitBlk)); + pIter->pBlock = (SSubmitBlk *)((char *)pBlock + pBlock->dataLen + sizeof(SSubmitBlk)); } return pBlock; @@ -832,10 +833,10 @@ _err: } static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { - if (pBlock->len <= 0) return -1; - pIter->totalLen = pBlock->len; + if (pBlock->dataLen <= 0) return -1; + pIter->totalLen = pBlock->dataLen; pIter->len = 0; - pIter->row = (SDataRow)(pBlock->data); + pIter->row = (SDataRow)(pBlock->data+pBlock->schemaLen); return 0; } From 457a701015d647bf966ce12d803d2998f1f25ae9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 30 Jun 2020 15:25:16 +0800 Subject: [PATCH 55/64] [td-808] --- src/cq/src/cqMain.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 769dfea774..098d69fcb2 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -283,7 +283,8 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { } tdAppendColVal(trow, val, c->type, c->bytes, c->offset); } - pBlk->len = htonl(dataRowLen(trow)); + pBlk->dataLen = htonl(dataRowLen(trow)); + pBlk->schemaLen = 0; pBlk->uid = htobe64(pObj->uid); pBlk->tid = htonl(pObj->tid); From 374bcb86f748613c3ff915ca5a8e6916ed0332db Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 30 Jun 2020 15:26:04 +0800 Subject: [PATCH 56/64] [td-225] refactor code. --- src/client/src/tscSubquery.c | 243 +++++++++++++++++--------------- src/common/inc/tglobal.h | 4 +- src/common/src/tglobal.c | 6 +- src/os/linux/src/linuxSysPara.c | 4 +- src/query/tests/astTest.cpp | 2 +- 5 files changed, 137 insertions(+), 122 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index b9ba22436e..0243d115f0 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1415,7 +1415,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { } trs->subqueryIndex = i; - trs->pParentSqlObj = pSql; + trs->pParentSql = pSql; trs->pFinalColModel = pModel; pthread_mutexattr_t mutexattr; @@ -1500,7 +1500,7 @@ static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES tscError("sub:%p failed to flush data to disk, reason:%s", tres, tstrerror(code)); #endif - SSqlObj* pParentSql = trsupport->pParentSqlObj; + SSqlObj* pParentSql = trsupport->pParentSql; pParentSql->res.code = code; trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; @@ -1509,8 +1509,45 @@ static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES tscHandleSubqueryError(trsupport, tres, pParentSql->res.code); } +/* + * current query failed, and the retry count is less than the available + * count, retry query clear previous retrieved data, then launch a new sub query + */ +static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, int32_t code) { + SSqlObj *pParentSql = trsupport->pParentSql; + int32_t subqueryIndex = trsupport->subqueryIndex; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); + SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; + + tExtMemBufferClear(trsupport->pExtMemBuffer[subqueryIndex]); + + // clear local saved number of results + trsupport->localBuffer->num = 0; + pthread_mutex_unlock(&trsupport->queryMutex); + + tscTrace("%p sub:%p retrieve failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, + tstrerror(code), subqueryIndex, trsupport->numOfRetry); + + SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSql, trsupport, pSql); + + // todo add to async res or not?? + if (pNew == NULL) { + tscError("%p sub:%p failed to create new subquery due to out of memory, abort retry, vgId:%d, orderOfSub:%d", + trsupport->pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex); + + pParentSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; + trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; + + return pParentSql->res.code; + } + + taos_free_result(pSql); + return tscProcessSql(pNew); +} + void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) { - SSqlObj *pParentSql = trsupport->pParentSqlObj; + SSqlObj *pParentSql = trsupport->pParentSql; int32_t subqueryIndex = trsupport->subqueryIndex; assert(pSql != NULL); @@ -1529,38 +1566,16 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO tscDebug("%p query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%d", pParentSql, pSql, subqueryIndex, pParentSql->res.code); } - + if (numOfRows >= 0) { // current query is successful, but other sub query failed, still abort current query. tscDebug("%p sub:%p retrieve numOfRows:%d,orderOfSub:%d", pParentSql, pSql, numOfRows, subqueryIndex); tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%d", pParentSql, pSql, subqueryIndex, pParentSql->res.code); } else { if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && pParentSql->res.code == TSDB_CODE_SUCCESS) { - /* - * current query failed, and the retry count is less than the available - * count, retry query clear previous retrieved data, then launch a new sub query - */ - tExtMemBufferClear(trsupport->pExtMemBuffer[subqueryIndex]); - - // clear local saved number of results - trsupport->localBuffer->num = 0; - pthread_mutex_unlock(&trsupport->queryMutex); - - tscDebug("%p sub:%p retrieve failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSqlObj, pSql, - tstrerror(numOfRows), subqueryIndex, trsupport->numOfRetry); - - SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSqlObj, trsupport, pSql); - if (pNew == NULL) { - tscError("%p sub:%p failed to create new subquery sqlObj due to out of memory, abort retry", - trsupport->pParentSqlObj, pSql); - - pParentSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; + if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { return; } - - tscProcessSql(pNew); - return; } else { // reach the maximum retry count, abort atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, numOfRows); tscError("%p sub:%p retrieve failed,code:%s,orderOfSub:%d failed.no more retry,set global code:%s", pParentSql, pSql, @@ -1601,7 +1616,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* pSql) { int32_t idx = trsupport->subqueryIndex; - SSqlObj * pPObj = trsupport->pParentSqlObj; + SSqlObj * pParentSql = trsupport->pParentSql; tOrderDescriptor *pDesc = trsupport->pOrderDescriptor; SSubqueryState* pState = trsupport->pState; @@ -1611,7 +1626,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p // data in from current vnode is stored in cache and disk uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->num; - tscDebug("%p sub:%p all data retrieved from ip:%s, vgId:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, + tscDebug("%p sub:%p all data retrieved from ip:%s, vgId:%d, numOfRows:%d, orderOfSub:%d", pParentSql, pSql, pTableMetaInfo->vgroupList->vgroups[0].ipAddr[0].fqdn, pTableMetaInfo->vgroupList->vgroups[0].vgId, numOfRowsFromSubquery, idx); @@ -1625,15 +1640,14 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p trsupport->localBuffer->num, colInfo); #endif - if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) { - tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql, - tsAvailTmpDirGB, tsMinimalTmpDirGB); - tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_TSC_NO_DISKSPACE); - return; + if (tsTotalTmpDirGB != 0 && tsAvailTmpDirectorySpace < tsReservedTmpDirectorySpace) { + tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pParentSql, pSql, + tsAvailTmpDirectorySpace, tsReservedTmpDirectorySpace); + return tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_TSC_NO_DISKSPACE); } // each result for a vnode is ordered as an independant list, - // then used as an input of loser tree for disk-based merge routine + // then used as an input of loser tree for disk-based merge int32_t code = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pQueryInfo->groupbyExpr.orderType); if (code != 0) { // set no disk space error info, and abort retry return tscAbortFurtherRetryRetrieval(trsupport, pSql, code); @@ -1641,7 +1655,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p int32_t remain = -1; if ((remain = atomic_sub_fetch_32(&pState->numOfRemain, 1)) > 0) { - tscDebug("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, + tscDebug("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pParentSql, pSql, trsupport->subqueryIndex, pState->numOfTotal - remain); return tscFreeSubSqlObj(trsupport, pSql); @@ -1650,29 +1664,29 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p // all sub-queries are returned, start to local merge process pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; - tscDebug("%p retrieve from %d vnodes completed.final NumOfRows:%" PRId64 ",start to build loser tree", pPObj, + tscDebug("%p retrieve from %d vnodes completed.final NumOfRows:%" PRId64 ",start to build loser tree", pParentSql, pState->numOfTotal, pState->numOfRetrievedRows); - SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0); + SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, 0); tscClearInterpInfo(pPQueryInfo); - tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, pPObj); - tscDebug("%p build loser tree completed", pPObj); + tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, pParentSql); + tscDebug("%p build loser tree completed", pParentSql); - pPObj->res.precision = pSql->res.precision; - pPObj->res.numOfRows = 0; - pPObj->res.row = 0; + pParentSql->res.precision = pSql->res.precision; + pParentSql->res.numOfRows = 0; + pParentSql->res.row = 0; // only free once tfree(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); // set the command flag must be after the semaphore been correctly set. - pPObj->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE; - if (pPObj->res.code == TSDB_CODE_SUCCESS) { - (*pPObj->fp)(pPObj->param, pPObj, 0); + pParentSql->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE; + if (pParentSql->res.code == TSDB_CODE_SUCCESS) { + (*pParentSql->fp)(pParentSql->param, pParentSql, 0); } else { - tscQueueAsyncRes(pPObj); + tscQueueAsyncRes(pParentSql); } } @@ -1680,22 +1694,48 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR SRetrieveSupport *trsupport = (SRetrieveSupport *)param; tOrderDescriptor *pDesc = trsupport->pOrderDescriptor; int32_t idx = trsupport->subqueryIndex; - SSqlObj * pPObj = trsupport->pParentSqlObj; + SSqlObj * pParentSql = trsupport->pParentSql; SSqlObj *pSql = (SSqlObj *)tres; if (pSql == NULL) { // sql object has been released in error process, return immediately - tscDebug("%p subquery has been released, idx:%d, abort", pPObj, idx); + tscDebug("%p subquery has been released, idx:%d, abort", pParentSql, idx); return; } SSubqueryState* pState = trsupport->pState; - assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pPObj->numOfSubs == pState->numOfTotal); + assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pParentSql->numOfSubs == pState->numOfTotal); // query process and cancel query process may execute at the same time pthread_mutex_lock(&trsupport->queryMutex); - - if (numOfRows < 0 || pPObj->res.code != TSDB_CODE_SUCCESS) { - return tscHandleSubqueryError(trsupport, pSql, numOfRows); + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); + SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; + + if (pParentSql->res.code != TSDB_CODE_SUCCESS) { + trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; + tscTrace("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", + pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(numOfRows), tstrerror(pParentSql->res.code)); + + tscHandleSubqueryError(param, tres, numOfRows); + return; + } + + if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { + assert(numOfRows == taos_errno(pSql)); + + if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { + tscTrace("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); + + if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { + return; + } + } else { + tscTrace("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(numOfRows)); + atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, numOfRows); // set global code and abort + } + + tscHandleSubqueryError(param, tres, numOfRows); + return; } SSqlRes * pRes = &pSql->res; @@ -1705,14 +1745,13 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR assert(pRes->numOfRows == numOfRows); int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows); - tscDebug("%p sub:%p retrieve numOfRows:%" PRId64 " totalNumOfRows:%" PRIu64 " from ip:%s, orderOfSub:%d", pPObj, pSql, + tscDebug("%p sub:%p retrieve numOfRows:%" PRId64 " totalNumOfRows:%" PRIu64 " from ip:%s, orderOfSub:%d", pParentSql, pSql, pRes->numOfRows, pState->numOfRetrievedRows, pSql->ipList.fqdn[pSql->ipList.inUse], idx); if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId32 " , current:%" PRId64, - pPObj, pSql, tsMaxNumOfOrderedResults, num); - tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_SORTED_RES_TOO_MANY); - return; + pParentSql, pSql, tsMaxNumOfOrderedResults, num); + return tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_SORTED_RES_TOO_MANY); } #ifdef _DEBUG_VIEW @@ -1723,11 +1762,11 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR tColModelDisplayEx(pDesc->pColumnModel, pRes->data, pRes->numOfRows, pRes->numOfRows, colInfo); #endif - if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) { - tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql, - tsAvailTmpDirGB, tsMinimalTmpDirGB); - tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_NO_DISKSPACE); - return; + // no disk space for tmp directory + if (tsTotalTmpDirGB != 0 && tsAvailTmpDirectorySpace < tsReservedTmpDirectorySpace) { + tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pParentSql, pSql, + tsAvailTmpDirectorySpace, tsReservedTmpDirectorySpace); + return tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_NO_DISKSPACE); } int32_t ret = saveToBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pRes->data, @@ -1772,80 +1811,56 @@ static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsu void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { SRetrieveSupport *trsupport = (SRetrieveSupport *) param; - SSqlObj* pParentSql = trsupport->pParentSqlObj; + SSqlObj* pParentSql = trsupport->pParentSql; SSqlObj* pSql = (SSqlObj *) tres; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); assert(pSql->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; - - SSubqueryState* pState = trsupport->pState; - assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pParentSql->numOfSubs == pState->numOfTotal); + SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[trsupport->subqueryIndex]; - // todo set error code + // stable query killed or other subquery failed, all query stopped if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - - // stable query is killed, abort further retry trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - code = pParentSql->res.code; - } - - tscDebug("%p query cancelled or failed, sub:%p, orderOfSub:%d abort, code:%s", pParentSql, pSql, - trsupport->subqueryIndex, tstrerror(code)); + tscTrace("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", + pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(code), tstrerror(pParentSql->res.code)); + + tscHandleSubqueryError(param, tres, code); + return; } /* - * if a query on a vnode is failed, all retrieve operations from vnode that occurs later + * if a subquery on a vnode failed, all retrieve operations from vnode that occurs later * than this one are actually not necessary, we simply call the tscRetrieveFromDnodeCallBack * function to abort current and remain retrieve process. * * NOTE: thread safe is required. */ - if (code != TSDB_CODE_SUCCESS) { - if (trsupport->numOfRetry++ >= MAX_NUM_OF_SUBQUERY_RETRY) { - tscDebug("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(code)); - atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, code); - } else { // does not reach the maximum retry time, go on - tscDebug("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); - - SSqlObj *pNew = tscCreateSqlObjForSubquery(pParentSql, trsupport, pSql); + if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { + assert(code == taos_errno(pSql)); - if (pNew == NULL) { - tscError("%p sub:%p failed to create new subquery due to out of memory, abort retry, vgId:%d, orderOfSub:%d", - trsupport->pParentSqlObj, pSql, pVgroup->vgId, trsupport->subqueryIndex); - - pParentSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - } else { - SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); - assert(pNewQueryInfo->pTableMetaInfo[0]->pTableMeta != NULL); - - taos_free_result(pSql); - tscProcessSql(pNew); + if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { + tscTrace("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); + if (tscReissueSubquery(trsupport, pSql, code) == TSDB_CODE_SUCCESS) { return; } - } - } - - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { // at least one peer subquery failed, abort current query - tscDebug("%p sub:%p query failed,ip:%s,vgId:%d,orderOfSub:%d,global code:%d", pParentSql, pSql, - pVgroup->ipAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex, pParentSql->res.code); - - tscHandleSubqueryError(param, tres, pParentSql->res.code); - } else { // success, proceed to retrieve data from dnode - tscDebug("%p sub:%p query complete, ip:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSqlObj, pSql, - pVgroup->ipAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex); - - if (pSql->res.qhandle == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode - tscRetrieveFromDnodeCallBack(param, pSql, 0); } else { - taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); + tscTrace("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(code)); + atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, code); // set global code and abort } - + + tscHandleSubqueryError(param, tres, pParentSql->res.code); + return; + } + + tscTrace("%p sub:%p query complete, ip:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql, + pVgroup->ipAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex); + + if (pSql->res.qhandle == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode + tscRetrieveFromDnodeCallBack(param, pSql, 0); + } else { + taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); } } diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 371df5c335..af1d7dd441 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -128,10 +128,10 @@ extern float tsTotalLogDirGB; extern float tsTotalTmpDirGB; extern float tsTotalDataDirGB; extern float tsAvailLogDirGB; -extern float tsAvailTmpDirGB; +extern float tsAvailTmpDirectorySpace; extern float tsAvailDataDirGB; extern float tsMinimalLogDirGB; -extern float tsMinimalTmpDirGB; +extern float tsReservedTmpDirectorySpace; extern float tsMinimalDataDirGB; extern int32_t tsTotalMemoryMB; extern int32_t tsVersion; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 6dc6a82d09..90db4683d8 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -170,9 +170,9 @@ int64_t tsStreamMax; int32_t tsNumOfCores = 1; float tsTotalTmpDirGB = 0; float tsTotalDataDirGB = 0; -float tsAvailTmpDirGB = 0; +float tsAvailTmpDirectorySpace = 0; float tsAvailDataDirGB = 0; -float tsMinimalTmpDirGB = 0.1; +float tsReservedTmpDirectorySpace = 0.1; float tsMinimalDataDirGB = 0.5; int32_t tsTotalMemoryMB = 0; int32_t tsVersion = 0; @@ -807,7 +807,7 @@ static void doInitGlobalConfig() { taosInitConfigOption(cfg); cfg.option = "minimalTmpDirGB"; - cfg.ptr = &tsMinimalTmpDirGB; + cfg.ptr = &tsReservedTmpDirectorySpace; cfg.valType = TAOS_CFG_VTYPE_FLOAT; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0.001; diff --git a/src/os/linux/src/linuxSysPara.c b/src/os/linux/src/linuxSysPara.c index e4239c11e5..c2134765df 100644 --- a/src/os/linux/src/linuxSysPara.c +++ b/src/os/linux/src/linuxSysPara.c @@ -326,12 +326,12 @@ bool taosGetDisk() { if (statvfs("/tmp", &info)) { //tsTotalTmpDirGB = 0; - //tsAvailTmpDirGB = 0; + //tsAvailTmpDirectorySpace = 0; uError("failed to get disk size, tmpDir:/tmp errno:%s", strerror(errno)); return false; } else { tsTotalTmpDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit); - tsAvailTmpDirGB = (float)((double)info.f_bavail * (double)info.f_frsize / unit); + tsAvailTmpDirectorySpace = (float)((double)info.f_bavail * (double)info.f_frsize / unit); } return true; diff --git a/src/query/tests/astTest.cpp b/src/query/tests/astTest.cpp index d767e7ad7b..15eb780021 100644 --- a/src/query/tests/astTest.cpp +++ b/src/query/tests/astTest.cpp @@ -631,5 +631,5 @@ void exprSerializeTest2() { } } // namespace TEST(testCase, astTest) { - exprSerializeTest2(); +// exprSerializeTest2(); } \ No newline at end of file From eec38d8699aac656f79f04310cec305b34360490 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 30 Jun 2020 18:36:38 +0800 Subject: [PATCH 57/64] [td-805] opt query perf. --- src/client/src/tscParseInsert.c | 2 -- src/query/src/qExecutor.c | 19 +++++++++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 6fac028049..f0c8ffebc2 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1259,8 +1259,6 @@ int tsParseInsertSql(SSqlObj *pSql) { if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { goto _error; } - } else { - pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); } code = TSDB_CODE_SUCCESS; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 35880232f9..9df161b3dc 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3270,6 +3270,8 @@ static bool hasMainOutput(SQuery *pQuery) { } static STableQueryInfo *createTableQueryInfo( SQueryRuntimeEnv *pRuntimeEnv, void* pTable, STimeWindow win) { + SQuery* pQuery = pRuntimeEnv->pQuery; + STableQueryInfo *pTableQueryInfo = calloc(1, sizeof(STableQueryInfo)); pTableQueryInfo->win = win; @@ -3278,7 +3280,15 @@ static STableQueryInfo *createTableQueryInfo( SQueryRuntimeEnv *pRuntimeEnv, voi pTableQueryInfo->pTable = pTable; pTableQueryInfo->cur.vgroupIndex = -1; - initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, 100, 100, TSDB_DATA_TYPE_INT); + int32_t initialSize = 1; + int32_t initialThreshold = 1; + + if (isIntervalQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + initialSize = 20; + initialThreshold = 100; + } + + initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, initialSize, initialThreshold, TSDB_DATA_TYPE_INT); return pTableQueryInfo; } @@ -3310,6 +3320,10 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { STableQueryInfo *pTableQueryInfo = pRuntimeEnv->pQuery->current; SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + // lastKey needs to be updated + pTableQueryInfo->lastKey = nextKey; + setAdditionalInfo(pQInfo, pTableQueryInfo->pTable, pTableQueryInfo); + if (pRuntimeEnv->prevGroupId != INT32_MIN && pRuntimeEnv->prevGroupId == groupIndex) { return; } @@ -3335,9 +3349,6 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { pRuntimeEnv->prevGroupId = groupIndex; setWindowResOutputBuf(pRuntimeEnv, pWindowRes); initCtxOutputBuf(pRuntimeEnv); - - pTableQueryInfo->lastKey = nextKey; - setAdditionalInfo(pQInfo, pTableQueryInfo->pTable, pTableQueryInfo); } void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) { From a43544ea266eab6a2e9321fffb663e1a647cd275 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 30 Jun 2020 14:31:06 +0000 Subject: [PATCH 58/64] invalid read while auto create table --- src/inc/taoserror.h | 1 + src/mnode/src/mnodeTable.c | 16 +++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 33e0aa24b6..4a7d86c434 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -153,6 +153,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREAY_EXIST, 0, 0x0369, "mnode tag TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST, 0, 0x036A, "mnode tag not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_ALREAY_EXIST, 0, 0x036B, "mnode field already exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST, 0, 0x036C, "mnode field not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME, 0, 0x036D, "mnode invalid stable name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, 0, 0x0380, "mnode db not selected") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, 0, 0x0381, "mnode database aleady exist") diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 65900a60db..23b62f1aa3 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1964,9 +1964,15 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { SCMTableInfoMsg *pInfo = pMsg->rpcMsg.pCont; - STagData *pTag = (STagData *)pInfo->tags; + STagData *pTags = (STagData *)pInfo->tags; + int32_t tagLen = htonl(pTags->dataLen); + if (pTags->name[0] == 0) { + mError("app:%p:%p, table:%s, failed to create table on demand for stable is empty, tagLen:%d", pMsg->rpcMsg.ahandle, + pMsg, pInfo->tableId, tagLen); + return TSDB_CODE_MND_INVALID_STABLE_NAME; + } - int32_t contLen = sizeof(SCMCreateTableMsg) + offsetof(STagData, data) + htonl(pTag->dataLen); + int32_t contLen = sizeof(SCMCreateTableMsg) + offsetof(STagData, data) + tagLen; SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen); if (pCreateMsg == NULL) { mError("app:%p:%p, table:%s, failed to create table while get meta info, no enough memory", pMsg->rpcMsg.ahandle, @@ -1981,9 +1987,9 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { pCreateMsg->getMeta = 1; pCreateMsg->contLen = htonl(contLen); - memcpy(pCreateMsg->schema, pInfo->tags, contLen - sizeof(SCMCreateTableMsg)); - mDebug("app:%p:%p, table:%s, start to create on demand, stable:%s", pMsg->rpcMsg.ahandle, pMsg, pInfo->tableId, - ((STagData *)(pCreateMsg->schema))->name); + memcpy(pCreateMsg->schema, pTags, contLen); + mDebug("app:%p:%p, table:%s, start to create on demand, tagLen:%d stable:%s", + pMsg->rpcMsg.ahandle, pMsg, pInfo->tableId, tagLen, pTags->name); rpcFreeCont(pMsg->rpcMsg.pCont); pMsg->rpcMsg.msgType = TSDB_MSG_TYPE_CM_CREATE_TABLE; From 24eae824ecec5b7239a9cbf2346000929593745c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 30 Jun 2020 23:19:00 +0800 Subject: [PATCH 59/64] Invalid write while create child ctable --- src/mnode/src/mnodeTable.c | 6 ++++-- src/plugins/http/src/httpSql.c | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 23b62f1aa3..82322d43b8 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -382,11 +382,13 @@ static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCt pStable->numOfTables++; if (pStable->vgHash == NULL) { - pStable->vgHash = taosHashInit(100000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); } if (pStable->vgHash != NULL) { - taosHashPut(pStable->vgHash, (char *)&pCtable->vgId, sizeof(pCtable->vgId), &pCtable->vgId, sizeof(pCtable->vgId)); + if (taosHashGet(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId)) == NULL) { + taosHashPut(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId), &pCtable->vgId, sizeof(pCtable->vgId)); + } } } diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index d0e9b005fd..9d3efca01d 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -359,6 +359,8 @@ void httpExecCmd(HttpContext *pContext) { void httpProcessRequestCb(void *param, TAOS_RES *result, int code) { HttpContext *pContext = param; + taos_free_result(result); + if (pContext == NULL) return; if (code < 0) { From 096b624340babe5bf5d03ce8d118b356395ca3b9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 30 Jun 2020 15:20:27 +0000 Subject: [PATCH 60/64] crash while calc tagLen --- src/mnode/src/mnodeTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 23b62f1aa3..5ebfb164df 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1987,7 +1987,7 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { pCreateMsg->getMeta = 1; pCreateMsg->contLen = htonl(contLen); - memcpy(pCreateMsg->schema, pTags, contLen); + memcpy(pCreateMsg->schema, pTags, contLen - sizeof(SCMCreateTableMsg)); mDebug("app:%p:%p, table:%s, start to create on demand, tagLen:%d stable:%s", pMsg->rpcMsg.ahandle, pMsg, pInfo->tableId, tagLen, pTags->name); From 505b9a909161aef7f016f5def3a9e15700c75b5f Mon Sep 17 00:00:00 2001 From: dengyihao Date: Wed, 1 Jul 2020 02:38:17 +0800 Subject: [PATCH 61/64] fix test case --- tests/pytest/query/queryJoin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/pytest/query/queryJoin.py b/tests/pytest/query/queryJoin.py index bfc493580e..6d135e1006 100644 --- a/tests/pytest/query/queryJoin.py +++ b/tests/pytest/query/queryJoin.py @@ -112,8 +112,7 @@ class TDTestCase: tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id") tdSql.checkRows(6) - tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.pid, stb_p.id, stb_p.dscrption, stb_p.pressure,stb_v.velocity from stb_p, stb_t, stb_v where stb_p.ts=stb_t.ts and stb_p.ts=stb_v.ts and stb_p.id = stb_t.id") - tdSql.checkRows(2) + tdSql.error("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.pid, stb_p.id, stb_p.dscrption, stb_p.pressure,stb_v.velocity from stb_p, stb_t, stb_v where stb_p.ts=stb_t.ts and stb_p.ts=stb_v.ts and stb_p.id = stb_t.id") def stop(self): tdSql.close() From 37743a1fbf841b2732a9278bf684968eb7152b80 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 1 Jul 2020 10:08:11 +0800 Subject: [PATCH 62/64] [TD-811] --- src/kit/taosdemo/taosdemo.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 3e802c9566..3265285cca 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -808,27 +808,31 @@ void *readMetric(void *sarg) { } void queryDB(TAOS *taos, char *command) { - int i = 5; + int i; TAOS_RES *pSql = NULL; - int32_t code = -1; - while (i > 0 && code != 0) { + int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != pSql) { + taos_free_result(pSql); + pSql = NULL; + } + pSql = taos_query(taos, command); code = taos_errno(pSql); - taos_free_result(pSql); - pSql = NULL; - if (code == 0) { + if (0 == code) { break; - } - i--; + } } if (code != 0) { fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql)); taos_free_result(pSql); - taos_close(taos); exit(EXIT_FAILURE); } + + taos_free_result(pSql); } // sync insertion From c14562506c6f72d84de94b4dc7bd37cc18eb9d85 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 1 Jul 2020 11:33:46 +0800 Subject: [PATCH 63/64] update perf-test script. --- tests/perftest-scripts/perftest-daily.sh | 127 +++++--- tests/perftest-scripts/perftest-taosdemo.sh | 73 +++-- .../perftest-tsdb-compare-13d.sh | 52 +++- .../perftest-tsdb-compare-1d.sh | 56 +++- .../perftest-tsdb-compare-var10k-int100s.sh | 84 ++++++ .../perftest-tsdb-compare-var10k-int10s.sh | 84 ++++++ tests/perftest-scripts/runInfluxdb.sh | 251 ++++++++++++++++ tests/perftest-scripts/runTDengine.sh | 283 ++++++++++++++++++ .../taosdemo-rps-csv2png.gnuplot | 24 ++ 9 files changed, 947 insertions(+), 87 deletions(-) create mode 100755 tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh create mode 100755 tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh create mode 100755 tests/perftest-scripts/runInfluxdb.sh create mode 100755 tests/perftest-scripts/runTDengine.sh create mode 100644 tests/perftest-scripts/taosdemo-rps-csv2png.gnuplot diff --git a/tests/perftest-scripts/perftest-daily.sh b/tests/perftest-scripts/perftest-daily.sh index b44387047c..beac8de15a 100755 --- a/tests/perftest-scripts/perftest-daily.sh +++ b/tests/perftest-scripts/perftest-daily.sh @@ -1,5 +1,7 @@ #!/bin/bash +WORK_DIR=/mnt/root + # Coloured Echoes # function red_echo { echo -e "\033[31m$@\033[0m"; } # function green_echo { echo -e "\033[32m$@\033[0m"; } # @@ -16,6 +18,17 @@ function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } # function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # +function setMaxTablesPerVnode { + echo "/etc/taos/taos.cfg maxTablesPerVnode will be set to $1" + + hasText=`grep "maxTablesPerVnode" /etc/taos/taos.cfg` + if [[ -z "$hasText" ]]; then + echo "maxTablesPerVnode $1" >> /etc/taos/taos.cfg + else + sed -i 's/^maxTablesPerVnode.*$/maxTablesPerVnode '"$1"'/g' /etc/taos/taos.cfg + fi +} + function setMaxConnections { echo "/etc/taos/taos.cfg maxConnection will be set to $1" @@ -27,6 +40,28 @@ function setMaxConnections { fi } +function setQDebugFlag { + echo "/etc/taos/taos.cfg qDebugFlag will be set to $1" + + hasText=`grep -w "qDebugFlag" /etc/taos/taos.cfg` + if [[ -z "$hasText" ]]; then + echo "qDebugFlag $1" >> /etc/taos/taos.cfg + else + sed -i 's/^qDebugFlag.*$/qDebugFlag '"$1"'/g' /etc/taos/taos.cfg + fi +} + +function setDebugFlag { + echo "/etc/taos/taos.cfg DebugFlag will be set to $1" + + hasText=`grep -w "DebugFlag" /etc/taos/taos.cfg` + if [[ -z "$hasText" ]]; then + echo "DebugFlag $1" >> /etc/taos/taos.cfg + else + sed -i 's/^DebugFlag.*$/DebugFlag '"$1"'/g' /etc/taos/taos.cfg + fi +} + function setWal { echo "/etc/taos/taos.cfg walLevel will be set to $1" @@ -47,9 +82,10 @@ function collectSysInfo { } function buildTDengine { - cd /root/TDengine + echoInfo "Build TDengine" + cd $WORK_DIR/TDengine - git remote update + git remote update > /dev/null REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop` LOCAL_COMMIT=`git rev-parse --short @` @@ -59,29 +95,17 @@ function buildTDengine { echo "repo up-to-date" else echo "repo need to pull" - git pull + git pull > /dev/null LOCAL_COMMIT=`git rev-parse --short @` cd debug rm -rf * - cmake .. + cmake .. > /dev/null make > /dev/null make install fi } -function restartTaosd { - systemctl stop taosd - pkill -KILL -x taosd - sleep 10 - - rm -rf /mnt/var/log/taos/* - rm -rf /mnt/var/lib/taos/* - - taosd 2>&1 > /dev/null & - sleep 10 -} - function sendReport { receiver="sdsang@taosdata.com, sangshuduo@gmail.com" mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" @@ -93,57 +117,90 @@ function sendReport { (cat - && uuencode perftest-13d-wal1-$today.log perftest-13d-wal1-$today.log)| \ (cat - && uuencode perftest-13d-wal1-report.csv perftest-13d-wal1-report-$today.csv) | \ (cat - && uuencode perftest-13d-wal1-report.png perftest-13d-wal1-report-$today.png) | \ + (cat - && uuencode perftest-var10k-int10s-wal1-$today.log perftest-var10k-int10s-wal1-$today.log)| \ + (cat - && uuencode perftest-var10k-int10s-wal1-report.csv perftest-var10k-int10s-wal1-report-$today.csv) | \ + (cat - && uuencode perftest-var10k-int10s-wal1-report.png perftest-var10k-int10s-wal1-report-$today.png) | \ (cat - && uuencode taosdemo-wal1-$today.log taosdemo-wal1-$today.log) | \ (cat - && uuencode taosdemo-wal1-report.csv taosdemo-wal1-report-$today.csv) | \ - (cat - && uuencode taosdemo-rps-wal1-report.csv taosdemo-rps-wal1-report-$today.csv) | \ (cat - && uuencode taosdemo-wal1-report.png taosdemo-wal1-report-$today.png) | \ + (cat - && uuencode taosdemo-rps-wal1-report.csv taosdemo-rps-wal1-report-$today.csv) | \ + (cat - && uuencode taosdemo-rps-wal1-report.png taosdemo-rps-wal1-report-$today.png) | \ (cat - && uuencode perftest-1d-wal2-$today.log perftest-1d-wal2-$today.log)| \ (cat - && uuencode perftest-1d-wal2-report.csv perftest-1d-wal2-report-$today.csv) | \ (cat - && uuencode perftest-1d-wal2-report.png perftest-1d-wal2-report-$today.png) | \ (cat - && uuencode perftest-13d-wal2-$today.log perftest-13d-wal2-$today.log)| \ (cat - && uuencode perftest-13d-wal2-report.csv perftest-13d-wal2-report-$today.csv) | \ (cat - && uuencode perftest-13d-wal2-report.png perftest-13d-wal2-report-$today.png) | \ + (cat - && uuencode perftest-var10k-int10s-wal2-$today.log perftest-var10k-int10s-wal2-$today.log)| \ + (cat - && uuencode perftest-var10k-int10s-wal2-report.csv perftest-var10k-int10s-wal2-report-$today.csv) | \ + (cat - && uuencode perftest-var10k-int10s-wal2-report.png perftest-var10k-int10s-wal2-report-$today.png) | \ (cat - && uuencode taosdemo-wal2-$today.log taosdemo-wal2-$today.log) | \ (cat - && uuencode taosdemo-wal2-report.csv taosdemo-wal2-report-$today.csv) | \ + (cat - && uuencode taosdemo-wal2-report.png taosdemo-wal2-report-$today.png) | \ (cat - && uuencode taosdemo-rps-wal2-report.csv taosdemo-rps-wal2-report-$today.csv) | \ + (cat - && uuencode taosdemo-rps-wal2-report.png taosdemo-rps-wal2-report-$today.png) | \ (cat - && uuencode sysinfo.log sysinfo.txt) | \ (cat - && uuencode taos.cfg taos-cfg-$today.txt) | \ ssmtp "${receiver}" } today=`date +"%Y%m%d"` -cd /root -echo -e "cron-ran-at-${today}" >> cron.log +cd $WORK_DIR +echo -e "cron-ran-at-${today}" >> $WORK_DIR/cron.log -echoInfo "Build TDengine" buildTDengine ############################ -setMaxConnections 100 +setMaxConnections 1000 +setMaxTablesPerVnode 6000 +setDebugFlag 131 +setQDebugFlag 131 ############################ setWal "2" -cd /root -./perftest-tsdb-compare-1d.sh "wal2" - -cd /root -./perftest-tsdb-compare-13d.sh "wal2" - -cd /root +cd $WORK_DIR +date >> $WORK_DIR/cron.log ./perftest-taosdemo.sh "wal2" +date >> $WORK_DIR/cron.log + +cd $WORK_DIR +date >> $WORK_DIR/cron.log +./perftest-tsdb-compare-1d.sh +date >> $WORK_DIR/cron.log + +cd $WORK_DIR +date >> $WORK_DIR/cron.log +./perftest-tsdb-compare-13d.sh +date >> $WORK_DIR/cron.log + +cd $WORK_DIR +date >> $WORK_DIR/cron.log +./perftest-tsdb-compare-var10k-int10s.sh +date >> $WORK_DIR/cron.log ############################# setWal "1" -cd /root -./perftest-tsdb-compare-1d.sh "wal1" - -cd /root -./perftest-tsdb-compare-13d.sh "wal1" - -cd /root +cd $WORK_DIR +date >> $WORK_DIR/cron.log ./perftest-taosdemo.sh "wal1" +date >> $WORK_DIR/cron.log + +cd $WORK_DIR +date >> $WORK_DIR/cron.log +./perftest-tsdb-compare-1d.sh +date >> $WORK_DIR/cron.log + +cd $WORK_DIR +date >> $WORK_DIR/cron.log +./perftest-tsdb-compare-13d.sh +date >> $WORK_DIR/cron.log + +cd $WORK_DIR +date >> $WORK_DIR/cron.log +./perftest-tsdb-compare-var10k-int10s.sh +date >> $WORK_DIR/cron.log ############################# collectSysInfo diff --git a/tests/perftest-scripts/perftest-taosdemo.sh b/tests/perftest-scripts/perftest-taosdemo.sh index 0dd9a0572a..e459769512 100755 --- a/tests/perftest-scripts/perftest-taosdemo.sh +++ b/tests/perftest-scripts/perftest-taosdemo.sh @@ -1,5 +1,20 @@ #!/bin/bash +WORK_DIR=/mnt/root + +walLevel=`grep "^walLevel" /etc/taos/taos.cfg | awk '{print $2}'` +if [[ "$walLevel" -eq "2" ]]; then + walPostfix="wal2" +elif [[ "$walLevel" -eq "1" ]]; then + walPostfix="wal1" +else + echo -e "${RED}wrong walLevel $walLevel found! ${NC}" + exit 1 +fi + +logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` +dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` + # Coloured Echoes function red_echo { echo -e "\033[31m$@\033[0m"; } function green_echo { echo -e "\033[32m$@\033[0m"; } @@ -17,13 +32,20 @@ function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } function restartTaosd { + echo "Stop taosd" systemctl stop taosd - pkill -KILL -x taosd - sleep 10 - - rm -rf /mnt/var/log/taos/* - rm -rf /mnt/var/lib/taos/* + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + + rm -rf $logDir/* + rm -rf $dataDir/* + echo "Start taosd" taosd 2>&1 > /dev/null & sleep 10 } @@ -32,7 +54,7 @@ function runCreateTableOnly { echoInfo "Restart Taosd" restartTaosd - /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -n 0 2>&1 | tee taosdemo-$1-$today.log" + /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -n 0 2>&1 | tee taosdemo-$walPostfix-$today.log" demoCreateTableOnly=`grep "Total:" totaltime.out|awk '{print $2}'` } @@ -40,7 +62,7 @@ function runDeleteTableOnly { echoInfo "Restart Taosd" restartTaosd - /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -t 0 -D 1 2>&1 | tee taosdemo-$1-$today.log" + /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -t 0 -D 1 2>&1 | tee taosdemo-$walPostfix-$today.log" demoDeleteTableOnly=`grep "Total:" totaltime.out|awk '{print $2}'` } @@ -48,41 +70,44 @@ function runCreateTableThenInsert { echoInfo "Restart Taosd" restartTaosd - /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo 2>&1 | tee -a taosdemo-$1-$today.log" + /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo 2>&1 | tee -a taosdemo-$walPostfix-$today.log" demoTableAndInsert=`grep "Total:" totaltime.out|awk '{print $2}'` - demoRPS=`grep "records\/second" taosdemo-$1-$today.log | tail -n1 | awk '{print $13}'` + demoRPS=`grep "records\/second" taosdemo-$walPostfix-$today.log | tail -n1 | awk '{print $13}'` } function generateTaosdemoPlot { - echo "${today} $1, demoCreateTableOnly: ${demoCreateTableOnly}, demoDeleteTableOnly: ${demoDeleteTableOnly}, demoTableAndInsert: ${demoTableAndInsert}" | tee -a taosdemo-$today.log - echo "${today}, ${demoCreateTableOnly}, ${demoDeleteTableOnly}, ${demoTableAndInsert}">> taosdemo-$1-report.csv - echo "${today}, ${demoRPS}" >> taosdemo-rps-$1-report.csv + echo "${today} $walPostfix, demoCreateTableOnly: ${demoCreateTableOnly}, demoDeleteTableOnly: ${demoDeleteTableOnly}, demoTableAndInsert: ${demoTableAndInsert}" | tee -a taosdemo-$today.log + echo "${today}, ${demoCreateTableOnly}, ${demoDeleteTableOnly}, ${demoTableAndInsert}">> taosdemo-$walPostfix-report.csv + echo "${today}, ${demoRPS}" >> taosdemo-rps-$walPostfix-report.csv - csvLines=`cat taosdemo-$1-report.csv | wc -l` + csvLines=`cat taosdemo-$walPostfix-report.csv | wc -l` if [ "$csvLines" -gt "10" ]; then - sed -i '1d' taosdemo-$1-report.csv + sed -i '1d' taosdemo-$walPostfix-report.csv fi - csvLines=`cat taosdemo-rps-$1-report.csv | wc -l` + csvLines=`cat taosdemo-rps-$walPostfix-report.csv | wc -l` if [ "$csvLines" -gt "10" ]; then - sed -i '1d' taosdemo-rps-$1-report.csv + sed -i '1d' taosdemo-rps-$walPostfix-report.csv fi - gnuplot -e "filename='taosdemo-$1-report'" -p taosdemo-csv2png.gnuplot - gnuplot -e "filename='taosdemo-rps-$1-report'" -p taosdemo-rps-csv2png.gnuplot + gnuplot -e "filename='taosdemo-$walPostfix-report'" -p taosdemo-csv2png.gnuplot + gnuplot -e "filename='taosdemo-rps-$walPostfix-report'" -p taosdemo-rps-csv2png.gnuplot } today=`date +"%Y%m%d"` -cd /root +cd $WORK_DIR echoInfo "Test Create Table Only " -runCreateTableOnly $1 +runCreateTableOnly echoInfo "Test Create Table then Insert data" -runDeleteTableOnly $1 +runDeleteTableOnly echoInfo "Test Create Table then Insert data" -runCreateTableThenInsert $1 +runCreateTableThenInsert echoInfo "Generate plot for taosdemo" -generateTaosdemoPlot $1 -echoInfo "End of TaosDemo Test" +generateTaosdemoPlot + +tar czf $WORK_DIR/taos-log-taosdemo-$today.tar.gz $logDir/* + +echoInfo "End of TaosDemo Test" | tee -a $WORK_DIR/cron.log diff --git a/tests/perftest-scripts/perftest-tsdb-compare-13d.sh b/tests/perftest-scripts/perftest-tsdb-compare-13d.sh index 8b326d2d41..110ab9e5fa 100755 --- a/tests/perftest-scripts/perftest-tsdb-compare-13d.sh +++ b/tests/perftest-scripts/perftest-tsdb-compare-13d.sh @@ -1,5 +1,21 @@ #!/bin/bash +WORK_DIR=/mnt/root +TSDB_CMP_DIR=timeseriesdatabase-comparisons/build/tsdbcompare + +walLevel=`grep "^walLevel" /etc/taos/taos.cfg | awk '{print $2}'` +if [[ "$walLevel" -eq "2" ]]; then + walPostfix="wal2" +elif [[ "$walLevel" -eq "1" ]]; then + walPostfix="wal1" +else + echo -e "${RED}wrong walLevel $walLevel found! ${NC}" + exit 1 +fi + +logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` +dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` + # Coloured Echoes # function red_echo { echo -e "\033[31m$@\033[0m"; } # function green_echo { echo -e "\033[32m$@\033[0m"; } # @@ -17,13 +33,20 @@ function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # function restartTaosd { + echo "Stop taosd" systemctl stop taosd - pkill -KILL -x taosd - sleep 10 - - rm -rf /mnt/var/log/taos/* - rm -rf /mnt/var/lib/taos/* + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + + rm -rf $logDir/* + rm -rf $dataDir/* + echo "Start taosd" taosd 2>&1 > /dev/null & sleep 10 } @@ -32,27 +55,30 @@ function runPerfTest13d { echoInfo "Restart Taosd" restartTaosd - cd /home/taos/tliu/timeseriesdatabase-comparisons/build/tsdbcompare - ./runreal-13d-csv.sh $1 2>&1 | tee /root/perftest-13d-$1-$today.log + cd $WORK_DIR/$TSDB_CMP_DIR + ./runTDengine.sh -d 13 -w -q 2>&1 | tee $WORK_DIR/perftest-13d-$walPostfix-$today.log } function generatePerfPlot13d { - cd /root + cd $WORK_DIR - csvLines=`cat perftest-13d-$1-report.csv | wc -l` + csvLines=`cat perftest-13d-$walPostfix-report.csv | wc -l` if [ "$csvLines" -gt "10" ]; then - sed -i '1d' perftest-13d-$1-report.csv + sed -i '1d' perftest-13d-$walPostfix-report.csv fi - gnuplot -e "filename='perftest-13d-$1-report'" -p perftest-csv2png.gnuplot + gnuplot -e "filename='perftest-13d-$walPostfix-report'" -p perftest-csv2png.gnuplot } today=`date +"%Y%m%d"` -cd /root +cd $WORK_DIR echoInfo "run Performance Test with 13 days data" runPerfTest13d $1 echoInfo "Generate plot of 13 days data" generatePerfPlot13d $1 -echoInfo "End of TSDB-Compare 13-days-data Test" + +tar czf $WORK_DIR/taos-log-13d-$today.tar.gz $logDir/* + +echoInfo "End of TSDB-Compare 13-days-data Test" | tee -a $WORK_DIR/cron.log diff --git a/tests/perftest-scripts/perftest-tsdb-compare-1d.sh b/tests/perftest-scripts/perftest-tsdb-compare-1d.sh index 0931480e21..9e8bc697bc 100755 --- a/tests/perftest-scripts/perftest-tsdb-compare-1d.sh +++ b/tests/perftest-scripts/perftest-tsdb-compare-1d.sh @@ -1,5 +1,21 @@ #!/bin/bash +WORK_DIR=/mnt/root +TSDB_CMP_DIR=timeseriesdatabase-comparisons/build/tsdbcompare + +walLevel=`grep "^walLevel" /etc/taos/taos.cfg | awk '{print $2}'` +if [[ "$walLevel" -eq "2" ]]; then + walPostfix="wal2" +elif [[ "$walLevel" -eq "1" ]]; then + walPostfix="wal1" +else + echo -e "${RED}wrong walLevel $walLevel found! ${NC}" + exit 1 +fi + +logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` +dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` + # Coloured Echoes # function red_echo { echo -e "\033[31m$@\033[0m"; } # function green_echo { echo -e "\033[32m$@\033[0m"; } # @@ -17,13 +33,20 @@ function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # function restartTaosd { + echo "Stop taosd" systemctl stop taosd - pkill -KILL -x taosd - sleep 10 - - rm -rf /mnt/var/log/taos/* - rm -rf /mnt/var/lib/taos/* + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + + rm -rf $logDir/* + rm -rf $dataDir/* + echo "Start taosd" taosd 2>&1 > /dev/null & sleep 10 } @@ -32,27 +55,30 @@ function runPerfTest1d { echoInfo "Restart Taosd" restartTaosd - cd /home/taos/tliu/timeseriesdatabase-comparisons/build/tsdbcompare - ./runreal-1d-csv.sh $1 2>&1 | tee /root/perftest-1d-$1-$today.log + cd $WORK_DIR/$TSDB_CMP_DIR + ./runTDengine.sh -d 1 -w -q 2>&1 | tee $WORK_DIR/perftest-1d-$walPostfix-$today.log } function generatePerfPlot1d { - cd /root + cd $WORK_DIR - csvLines=`cat perftest-1d-$1-report.csv | wc -l` + csvLines=`cat perftest-1d-$walPostfix-report.csv | wc -l` if [ "$csvLines" -gt "10" ]; then - sed -i '2d' perftest-1d-$1-report.csv + sed -i '1d' perftest-1d-$walPostfix-report.csv fi - gnuplot -e "filename='perftest-1d-$1-report'" -p perftest-csv2png.gnuplot + gnuplot -e "filename='perftest-1d-$walPostfix-report'" -p perftest-csv2png.gnuplot } today=`date +"%Y%m%d"` -cd /root +cd $WORK_DIR echoInfo "run Performance Test with 1 day data" -runPerfTest1d $1 +runPerfTest1d echoInfo "Generate plot of 1 day data" -generatePerfPlot1d $1 -echoInfo "End of TSDB-Compare 1-day-data Test" +generatePerfPlot1d + +tar czf $WORK_DIR/taos-log-1d-$today.tar.gz $logDir/* + +echoInfo "End of TSDB-Compare 1-day-data Test" | tee -a $WORK_DIR/cron.log diff --git a/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh b/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh new file mode 100755 index 0000000000..5fc5b9d03a --- /dev/null +++ b/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +WORK_DIR=/mnt/root +TSDB_CMP_DIR=timeseriesdatabase-comparisons/build/tsdbcompare + +walLevel=`grep "^walLevel" /etc/taos/taos.cfg | awk '{print $2}'` +if [[ "$walLevel" -eq "2" ]]; then + walPostfix="wal2" +elif [[ "$walLevel" -eq "1" ]]; then + walPostfix="wal1" +else + echo -e "${RED}wrong walLevel $walLevel found! ${NC}" + exit 1 +fi + +logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` +dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` + +# Coloured Echoes # +function red_echo { echo -e "\033[31m$@\033[0m"; } # +function green_echo { echo -e "\033[32m$@\033[0m"; } # +function yellow_echo { echo -e "\033[33m$@\033[0m"; } # +function white_echo { echo -e "\033[1;37m$@\033[0m"; } # +# Coloured Printfs # +function red_printf { printf "\033[31m$@\033[0m"; } # +function green_printf { printf "\033[32m$@\033[0m"; } # +function yellow_printf { printf "\033[33m$@\033[0m"; } # +function white_printf { printf "\033[1;37m$@\033[0m"; } # +# Debugging Outputs # +function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; } # +function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; } # +function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } # +function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # + +function restartTaosd { + echo "Stop taosd" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + + rm -rf $logDir/* + rm -rf $dataDir/* + + echo "Start taosd" + taosd 2>&1 > /dev/null & + sleep 10 +} + +function runPerfTestVar10K { + echoInfo "Restart Taosd" + restartTaosd + + cd $WORK_DIR/$TSDB_CMP_DIR + ./runTDengine.sh -v 10000 -i 100 -w -q 2>&1 | tee $WORK_DIR/perftest-var10k-int100s-$walPostfix-$today.log +} + +function generatePerfPlotVar10K { + cd $WORK_DIR + + csvLines=`cat perftest-var10k-int100s-$walPostfix-report.csv | wc -l` + + if [ "$csvLines" -gt "10" ]; then + sed -i '1d' perftest-var10k-int100s-$walPostfix-report.csv + fi + + gnuplot -e "filename='perftest-var10k-int100s-$walPostfix-report'" -p perftest-csv2png.gnuplot +} + +today=`date +"%Y%m%d"` +cd $WORK_DIR + +echoInfo "run Performance Test with 10K tables data" +runPerfTestVar10K +echoInfo "Generate plot of 10K tables data" +generatePerfPlotVar10K + +tar czf $WORK_DIR/taos-log-var10k-int100s-$today.tar.gz $logDir/* + +echoInfo "End of TSDB-Compare var10k-int100s-tables-data Test" | tee -a $WORK_DIR/cron.log diff --git a/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh b/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh new file mode 100755 index 0000000000..bafa04f174 --- /dev/null +++ b/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +WORK_DIR=/mnt/root +TSDB_CMP_DIR=timeseriesdatabase-comparisons/build/tsdbcompare + +walLevel=`grep "^walLevel" /etc/taos/taos.cfg | awk '{print $2}'` +if [[ "$walLevel" -eq "2" ]]; then + walPostfix="wal2" +elif [[ "$walLevel" -eq "1" ]]; then + walPostfix="wal1" +else + echo -e "${RED}wrong walLevel $walLevel found! ${NC}" + exit 1 +fi + +logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` +dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` + +# Coloured Echoes # +function red_echo { echo -e "\033[31m$@\033[0m"; } # +function green_echo { echo -e "\033[32m$@\033[0m"; } # +function yellow_echo { echo -e "\033[33m$@\033[0m"; } # +function white_echo { echo -e "\033[1;37m$@\033[0m"; } # +# Coloured Printfs # +function red_printf { printf "\033[31m$@\033[0m"; } # +function green_printf { printf "\033[32m$@\033[0m"; } # +function yellow_printf { printf "\033[33m$@\033[0m"; } # +function white_printf { printf "\033[1;37m$@\033[0m"; } # +# Debugging Outputs # +function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; } # +function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; } # +function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } # +function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # + +function restartTaosd { + echo "Stop taosd" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + + rm -rf $logDir/* + rm -rf $dataDir/* + + echo "Start taosd" + taosd 2>&1 > /dev/null & + sleep 10 +} + +function runPerfTestVar10K { + echoInfo "Restart Taosd" + restartTaosd + + cd $WORK_DIR/$TSDB_CMP_DIR + ./runTDengine.sh -v 10000 -w -q 2>&1 | tee $WORK_DIR/perftest-var10k-int10s-$walPostfix-$today.log +} + +function generatePerfPlotVar10K { + cd $WORK_DIR + + csvLines=`cat perftest-var10k-int10s-$walPostfix-report.csv | wc -l` + + if [ "$csvLines" -gt "10" ]; then + sed -i '1d' perftest-var10k-int10s-$walPostfix-report.csv + fi + + gnuplot -e "filename='perftest-var10k-int10s-$walPostfix-report'" -p perftest-csv2png.gnuplot +} + +today=`date +"%Y%m%d"` +cd $WORK_DIR + +echoInfo "run Performance Test with 10K tables data" +runPerfTestVar10K +echoInfo "Generate plot of 10K tables data" +generatePerfPlotVar10K + +tar czf $WORK_DIR/taos-log-var10k-int10s-$today.tar.gz $logDir/* + +echoInfo "End of TSDB-Compare var10k-int10s-tables-data Test" | tee -a $WORK_DIR/cron.log diff --git a/tests/perftest-scripts/runInfluxdb.sh b/tests/perftest-scripts/runInfluxdb.sh new file mode 100755 index 0000000000..e8975b05cf --- /dev/null +++ b/tests/perftest-scripts/runInfluxdb.sh @@ -0,0 +1,251 @@ +#!/bin/bash +#set -x + +WORK_DIR=/mnt/root +DATA_DIR=/mnt/data + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +# default value +DEFAULT_BATCH=5000 +DEFAULT_DAYS=1 +DEFAULT_INTERVAL=1 +DEFAULT_SCALEVAR=10 +DEFAULT_DOPREPARE=false +DEFAULT_DOWRITE=false +DEFAULT_DOQUERY=false + +# function +function do_prepare { + echo + echo "---------------Generating Data-----------------" + echo + + echo + echo "Prepare data for InfluxDB...." + + echo "bin/bulk_data_gen -seed 123 -format influx-bulk -sampling-interval $interval_s \ + -scale-var $scalevar -use-case devops -timestamp-start $TIME_START \ + -timestamp-end $TIME_END > $DATA_FILE" + + bin/bulk_data_gen -seed 123 -format influx-bulk -sampling-interval $interval_s \ + -scale-var $scalevar -use-case devops -timestamp-start $TIME_START \ + -timestamp-end $TIME_END > $DATA_FILE +} + +function do_write { + echo "cat $DATA_FILE | bin/bulk_load_influx \ + --batch-size=$batch --workers=20 --urls=http://172.15.1.5:8086 | grep loaded" + INFLUXRES=`cat $DATA_FILE | bin/bulk_load_influx \ + --batch-size=$batch --workers=20 --urls="http://172.15.1.5:8086" | grep loaded` + + echo -e "${GREEN}InfluxDB writing result:${NC}" + echo -e "${GREEN}$INFLUXRES${NC}" + DATA=`echo $INFLUXRES|awk '{print($2)}'` + TMP=`echo $INFLUXRES|awk '{print($5)}'` + IFWTM=`echo ${TMP%s*}` +} + +function do_query { + + echo + echo "------------------Querying Data-----------------" + echo + + echo + echo "start query test, query max from 8 hosts group by 1 hour, InfluxDB" + echo + +#Test case 1 +#测试用例1，查询所有数据中，用8个hostname标签进行匹配，匹配出这8个hostname对应的模拟服务器CPU数据中的usage_user这个监控数据的最大值。 +#select max(usage_user) from cpu where(hostname='host_a' and hostname='host_b'and hostname='host_c'and hostname='host_d'and hostname='host_e'and hostname='host_f' and hostname='host_g'and hostname='host_h') ; +# a,b,c,d,e,f,g,h are random 8 numbers. + echo "IFQS1=bin/bulk_query_gen -seed 123 -format influx-http -query-type 8-host-all -scale-var $scalevar -queries 1000 | bin/query_benchmarker_influxdb -urls="http://172.15.1.5:8086" -workers 50 -print-interval 0|grep wall" + + IFQS1=`bin/bulk_query_gen -seed 123 -format influx-http -query-type 8-host-all -scale-var $scalevar -queries 1000 | bin/query_benchmarker_influxdb -urls="http://172.15.1.5:8086" -workers 50 -print-interval 0|grep wall` + echo -e "${GREEN}InfluxDB query test case 1 result:${NC}" + echo -e "${GREEN}$IFQS1${NC}" + TMP=`echo $IFQS1|awk '{print($4)}'` + IFQ1=`echo ${TMP%s*}` + +#Test case 2 +#测试用例2，查询所有数据中，用8个hostname标签进行匹配，匹配出这8个hostname对应的模拟服务器CPU数据中的usage_user这个监控数据，以1小时为粒度，查询每1小时的最大值。 +#select max(usage_user) from cpu where(hostname='host_a' and hostname='host_b'and hostname='host_c'and hostname='host_d'and hostname='host_e'and hostname='host_f' and hostname='host_g'and hostname='host_h') interval(1h); +# a,b,c,d,e,f,g,h are random 8 numbers + echo "IFQS2=bin/bulk_query_gen -seed 123 -format influx-http -query-type 8-host-allbyhr -scale-var $scalevar -queries 1000 | bin/query_benchmarker_influxdb -urls=http://172.15.1.5:8086 -workers 50 -print-interval 0|grep wall" + + IFQS2=`bin/bulk_query_gen -seed 123 -format influx-http -query-type 8-host-allbyhr \ + -scale-var $scalevar -queries 1000 | bin/query_benchmarker_influxdb \ + -urls="http://172.15.1.5:8086" -workers 50 -print-interval 0|grep wall` + echo -e "${GREEN}InfluxDB query test case 2 result:${NC}" + echo -e "${GREEN}$IFQS2${NC}" + TMP=`echo $IFQS2|awk '{print($4)}'` + IFQ2=`echo ${TMP%s*}` + +#Test case 3 +#测试用例3，测试用例3，随机查询12个小时的数据，用8个hostname标签进行匹配，匹配出这8个hostname对应的模拟服务器CPU数据中的usage_user这个监控数据，以10分钟为粒度，查询每10分钟的最大值 +#select max(usage_user) from cpu where(hostname='host_a' and hostname='host_b'and hostname='host_c'and hostname='host_d'and hostname='host_e'and hostname='host_f' and hostname='host_g'and hostname='host_h') and time >x and time x and time >/dev/null 2>&1 +INFLUX=`docker run -d -p 8086:8086 --net tsdbcomp --ip 172.15.1.5 influxdb` >>/dev/null 2>&1 +sleep 10 + +if $dowrite; +then + echo -e "Start test InfluxDB writting, result in ${GREEN}Green line${NC}" + do_write +fi + +if $doquery; +then + echo -e "Start test InfluxDB query, result in ${GREEN}Green line${NC}" + do_query +fi + +echo +echo +echo "======================================================" +echo " tsdb performance comparision " +echo "======================================================" +if $dowrite; +then + echo -e " Writing $DATA records test takes: " + printf " InfluxDB | %-4.5f Seconds \n" $IFWTM + echo "------------------------------------------------------" +fi + +if $doquery; +then + echo " Query test cases: " + echo " case 1: select the max(value) from all data " + echo " filtered out 8 hosts " + echo " Query test case 1 takes: " + printf " InfluxDB | %-4.5f Seconds \n" $IFQ1 + echo "------------------------------------------------------" + echo " case 2: select the max(value) from all data " + echo " filtered out 8 hosts with an interval of 1 hour " + echo " case 2 takes: " + printf " InfluxDB | %-4.5f Seconds \n" $IFQ2 + echo "------------------------------------------------------" + echo " case 3: select the max(value) from random 12 hours" + echo " data filtered out 8 hosts with an interval of 10 min " + echo " filtered out 8 hosts interval(1h) " + echo " case 3 takes: " + printf " InfluxDB | %-4.5f Seconds \n" $IFQ3 + echo "------------------------------------------------------" + echo " case 4: select the max(value) from random 1 hour data " + echo " data filtered out 8 hosts with an interval of 1 min " + echo " case 4 takes: " + printf " InfluxDB | %-4.5f Seconds \n" $IFQ4 + echo "------------------------------------------------------" +fi + +docker stop $INFLUX >>/dev/null 2>&1 +docker container rm -f $INFLUX >>/dev/null 2>&1 +docker network rm tsdbcomp >>/dev/null 2>&1 + +today=`date +"%Y%m%d"` +echo "${today}, ${IFWTM}, ${IFQ1}, ${IFQ2}, ${IFQ3}, ${IFQ4}" >> $RECORD_CSV_FILE diff --git a/tests/perftest-scripts/runTDengine.sh b/tests/perftest-scripts/runTDengine.sh new file mode 100755 index 0000000000..6c5cf4e55e --- /dev/null +++ b/tests/perftest-scripts/runTDengine.sh @@ -0,0 +1,283 @@ +#!/bin/bash +#set -x + +WORK_DIR=/mnt/root +DATA_DIR=/mnt/data + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +# default value +DEFAULT_BATCH=5000 +DEFAULT_DAYS=1 +DEFAULT_INTERVAL=1 +DEFAULT_SCALEVAR=10 +DEFAULT_DOPREPARE=false +DEFAULT_DOWRITE=false +DEFAULT_DOQUERY=false + +# function +function do_prepare { + echo + echo "---------------Generating Data-----------------" + echo + + echo + echo "Prepare data for TDengine...." + + # bin/bulk_data_gen -seed 123 -format tdengine -tdschema-file config/TDengineSchema.toml -scale-var 100 -use-case devops -timestamp-start "2018-01-01T00:00:00Z" -timestamp-end "2018-01-02T00:00:00Z" > $DATA_DIR/tdengine.dat + echo "bin/bulk_data_gen -seed 123 -format tdengine -sampling-interval $interval_s \ + -tdschema-file config/TDengineSchema.toml -scale-var $scalevar \ + -use-case devops -timestamp-start $TIME_START \ + -timestamp-end $TIME_END \ + > $DATA_FILE" + + bin/bulk_data_gen -seed 123 -format tdengine -sampling-interval $interval_s \ + -tdschema-file config/TDengineSchema.toml -scale-var $scalevar \ + -use-case devops -timestamp-start $TIME_START \ + -timestamp-end $TIME_END \ + > $DATA_FILE +} + +function do_write { + echo "TDENGINERES=cat $DATA_FILE |bin/bulk_load_tdengine --url 127.0.0.1:0 \ + --batch-size $batch -do-load -report-tags n1 -workers 20 -fileout=false| grep loaded" + + TDENGINERES=`cat $DATA_FILE |bin/bulk_load_tdengine --url 127.0.0.1:0 \ + --batch-size $batch -do-load -report-tags n1 -workers 20 -fileout=false| grep loaded` + + echo + echo -e "${GREEN}TDengine writing result:${NC}" + echo -e "${GREEN}$TDENGINERES${NC}" + DATA=`echo $TDENGINERES|awk '{print($2)}'` + TMP=`echo $TDENGINERES|awk '{print($5)}'` + TDWTM=`echo ${TMP%s*}` + +} + +function do_query { + + echo + echo "------------------Querying Data-----------------" + echo + + echo + echo "start query test, query max from 8 hosts group by 1 hour, TDengine" + echo + +#Test case 1 +#测试用例1，查询所有数据中，用8个hostname标签进行匹配，匹配出这8个hostname对应的模拟服务器CPU数据中的usage_user这个监控数据的最大值。 +#select max(usage_user) from cpu where(hostname='host_a' and hostname='host_b'and hostname='host_c'and hostname='host_d'and hostname='host_e'and hostname='host_f' and hostname='host_g'and hostname='host_h') ; +# a,b,c,d,e,f,g,h are random 8 numbers. + echo "TDQS1=bin/bulk_query_gen -seed 123 -format tdengine -query-type 8-host-all \ + -scale-var $scalevar -queries 1000 | bin/query_benchmarker_tdengine \ + -urls=http://127.0.0.1:6020 -workers 50 -print-interval 0|grep wall" + + TDQS1=`bin/bulk_query_gen -seed 123 -format tdengine -query-type 8-host-all \ + -scale-var $scalevar -queries 1000 | bin/query_benchmarker_tdengine \ + -urls="http://127.0.0.1:6020" -workers 50 -print-interval 0|grep wall` + echo + echo -e "${GREEN}TDengine query test case 1 result:${NC}" + echo -e "${GREEN}$TDQS1${NC}" + TMP=`echo $TDQS1|awk '{print($4)}'` + TDQ1=`echo ${TMP%s*}` + +#Test case 2 +#测试用例2，查询所有数据中，用8个hostname标签进行匹配，匹配出这8个hostname对应的模拟服务器CPU数据中的usage_user这个监控数据，以1小时为粒度，查询每1小时的最大值。 +#select max(usage_user) from cpu where(hostname='host_a' and hostname='host_b'and hostname='host_c'and hostname='host_d'and hostname='host_e'and hostname='host_f' and hostname='host_g'and hostname='host_h') interval(1h); +# a,b,c,d,e,f,g,h are random 8 numbers + echo "TDQS2=bin/bulk_query_gen -seed 123 -format tdengine -query-type 8-host-allbyhr \ + -scale-var $scalevar -queries 1000 | bin/query_benchmarker_tdengine \ + -urls=http://127.0.0.1:6020 -workers 50 -print-interval 0|grep wall" + + TDQS2=`bin/bulk_query_gen -seed 123 -format tdengine -query-type 8-host-allbyhr \ + -scale-var $scalevar -queries 1000 | bin/query_benchmarker_tdengine \ + -urls="http://127.0.0.1:6020" -workers 50 -print-interval 0|grep wall` + + echo + echo -e "${GREEN}TDengine query test case 2 result:${NC}" + echo -e "${GREEN}$TDQS2${NC}" + TMP=`echo $TDQS2|awk '{print($4)}'` + TDQ2=`echo ${TMP%s*}` + +#Test case 3 +#测试用例3，测试用例3，随机查询12个小时的数据，用8个hostname标签进行匹配，匹配出这8个hostname对应的模拟服务器CPU数据中的usage_user这个监控数据，以10分钟为粒度，查询每10分钟的最大值 +#select max(usage_user) from cpu where(hostname='host_a' and hostname='host_b'and hostname='host_c'and hostname='host_d'and hostname='host_e'and hostname='host_f' and hostname='host_g'and hostname='host_h') and time >x and time x and time > $RECORD_CSV_FILE diff --git a/tests/perftest-scripts/taosdemo-rps-csv2png.gnuplot b/tests/perftest-scripts/taosdemo-rps-csv2png.gnuplot new file mode 100644 index 0000000000..a7fcb4bd43 --- /dev/null +++ b/tests/perftest-scripts/taosdemo-rps-csv2png.gnuplot @@ -0,0 +1,24 @@ +#!/user/bin/gnuplot +reset +set terminal png + +set title filename font ",20" + +set ylabel "Time in Seconds" + +set xdata time +set timefmt "%Y%m%d" +set format x "%Y-%m-%d" +set xlabel "Date" + +set style data linespoints + +set terminal pngcairo size 1024,768 enhanced font 'Segoe UI, 10' +set output filename . '.png' +set datafile separator ',' + +set key reverse Left outside +set grid + + +plot filename . '.csv' using 1:2 title "Request Per Second" From c43321fceafcf0a358fca0af9506e2a3a67f0e63 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 1 Jul 2020 11:53:48 +0800 Subject: [PATCH 64/64] fix a runtime error --- tests/pytest/query/queryMetaData.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/query/queryMetaData.py b/tests/pytest/query/queryMetaData.py index 01c63c3f4a..8fb9d9bf3a 100755 --- a/tests/pytest/query/queryMetaData.py +++ b/tests/pytest/query/queryMetaData.py @@ -58,7 +58,7 @@ class MetadataQuery: cursor.execute("use test") base = threadID * self.tables - tablesPerThread = (int) self.tables / self.numOfTherads + tablesPerThread = int (self.tables / self.numOfTherads) for i in range(tablesPerThread): cursor.execute( '''create table t%d using meters tags(