From ae9698e1c71b8509b291145b68094dac07121039 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 2 Jun 2022 12:52:16 +0000 Subject: [PATCH 01/63] feat: vnode multi-version --- source/dnode/vnode/src/inc/tsdb.h | 2 + source/dnode/vnode/src/tsdb/tsdbCommit.c | 107 +++++++------- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 154 ++++++++------------ 3 files changed, 120 insertions(+), 143 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index a62b4c4409..ab23b7506e 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -43,6 +43,8 @@ typedef struct SMemTable SMemTable; int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable); void tsdbMemTableDestroy2(SMemTable *pMemTable); +int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk); +int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); // tsdbMemTable ================ typedef struct STsdbRow STsdbRow; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 86929fe6d5..9e5a5a751b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -87,6 +87,7 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p static void tsdbLoadAndMergeFromCache(STsdb *pTsdb, SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, TSKEY maxKey, int maxRows, int8_t update); int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf); +static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn); int tsdbBegin(STsdb *pTsdb) { if (!pTsdb) return 0; @@ -100,57 +101,6 @@ int tsdbBegin(STsdb *pTsdb) { return 0; } -int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) { - SDiskID did; - SDFileSet nSet = {0}; - STsdbFS *pfs = REPO_FS(pRepo); - int level; - - ASSERT(pSet->fid >= pRtn->minFid); - - level = tsdbGetFidLevel(pSet->fid, pRtn); - - if (tfsAllocDisk(pRepo->pVnode->pTfs, level, &did) < 0) { - terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; - return -1; - } - - if (did.level > TSDB_FSET_LEVEL(pSet)) { - // Need to move the FSET to higher level - tsdbInitDFileSet(pRepo, &nSet, did, pSet->fid, FS_TXN_VERSION(pfs)); - - if (tsdbCopyDFileSet(pSet, &nSet) < 0) { - tsdbError("vgId:%d, failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid, - TSDB_FSET_LEVEL(pSet), did.level, tstrerror(terrno)); - return -1; - } - - if (tsdbUpdateDFileSet(pfs, &nSet) < 0) { - return -1; - } - - tsdbInfo("vgId:%d, FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid, - TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet), did.level, did.id); - } else { - // On a correct level - if (tsdbUpdateDFileSet(pfs, pSet) < 0) { - return -1; - } - } - - return 0; -} - -int tsdbPrepareCommit(STsdb *pTsdb) { - if (pTsdb->mem == NULL) return 0; - - ASSERT(pTsdb->imem == NULL); - - pTsdb->imem = pTsdb->mem; - pTsdb->mem = NULL; - return 0; -} - int tsdbCommit(STsdb *pRepo) { SCommitH commith = {0}; SDFileSet *pSet = NULL; @@ -223,6 +173,57 @@ int tsdbCommit(STsdb *pRepo) { return 0; } +static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) { + SDiskID did; + SDFileSet nSet = {0}; + STsdbFS *pfs = REPO_FS(pRepo); + int level; + + ASSERT(pSet->fid >= pRtn->minFid); + + level = tsdbGetFidLevel(pSet->fid, pRtn); + + if (tfsAllocDisk(pRepo->pVnode->pTfs, level, &did) < 0) { + terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; + return -1; + } + + if (did.level > TSDB_FSET_LEVEL(pSet)) { + // Need to move the FSET to higher level + tsdbInitDFileSet(pRepo, &nSet, did, pSet->fid, FS_TXN_VERSION(pfs)); + + if (tsdbCopyDFileSet(pSet, &nSet) < 0) { + tsdbError("vgId:%d, failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), did.level, tstrerror(terrno)); + return -1; + } + + if (tsdbUpdateDFileSet(pfs, &nSet) < 0) { + return -1; + } + + tsdbInfo("vgId:%d, FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet), did.level, did.id); + } else { + // On a correct level + if (tsdbUpdateDFileSet(pfs, pSet) < 0) { + return -1; + } + } + + return 0; +} + +int tsdbPrepareCommit(STsdb *pTsdb) { + if (pTsdb->mem == NULL) return 0; + + ASSERT(pTsdb->imem == NULL); + + pTsdb->imem = pTsdb->mem; + pTsdb->mem = NULL; + return 0; +} + void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) { STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo); TSKEY minKey, midKey, maxKey, now; @@ -543,8 +544,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid return -1; } - tsdbDebug("vgId:%d, FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo), TSDB_FSET_FID(pSet), - TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + tsdbDebug("vgId:%d, FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo), + TSDB_FSET_FID(pSet), TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); } else { pCommith->isRFileSet = false; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index f4d4d94142..3bad5fdd92 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -64,9 +64,10 @@ static int memDataPCmprFn(const void *p1, const void *p2); static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow); static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl); -static void memDataMovePos(SMemData *pMemData, TSDBROW *pRow, int8_t isForward, SMemSkipListNode **pos); -static int32_t memDataPutRow(SVBufPool *pPool, SMemData *pMemData, TSDBROW *pRow, int8_t isForward, - SMemSkipListNode **pos); +static void memDataGetPosToPut(SMemData *pMemData, SMemSkipListNode **backward, TSDBROW *pRow); +static int32_t memDataDoPut(SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow, int8_t isForward); +static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, int64_t version, + SVSubmitBlk *pSubmitBlk); // SMemTable ============================================== int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) { @@ -123,28 +124,9 @@ int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmit } // do insert - int32_t nt; - int32_t n = 0; - uint8_t *p = pSubmitBlk->pData; - int32_t nRow = 0; - SMemSkipListNode *pos[SL_MAX_LEVEL] = {0}; - - for (int8_t iLevel = 0; iLevel < SL_MAX_LEVEL; iLevel++) { - pos[iLevel] = pMemData->sl.pTail; - } - while (n < pSubmitBlk->nData) { - nt = tGetTSRow(p + n, &row.tsRow); - n += nt; - - ASSERT(n <= pSubmitBlk->nData); - - memDataMovePos(pMemData, &row, nRow ? 1 : 0, pos); - code = memDataPutRow(pTsdb->pVnode->inUse, pMemData, &row, nRow ? 1 : 0, pos); - if (code) { - goto _err; - } - - nRow++; + code = tsdbInsertTableDataImpl(pMemTable, pMemData, version, pSubmitBlk); + if (code) { + goto _err; } return code; @@ -310,87 +292,79 @@ static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { return level; } -static void memDataMovePos(SMemData *pMemData, TSDBROW *pRow, int8_t isForward, SMemSkipListNode **pos) { - TSDBKEY *pKey; - int c; +static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, int64_t version, + SVSubmitBlk *pSubmitBlk) { + int32_t code = 0; + int32_t n = 0; + uint8_t *p = pSubmitBlk->pData; + int32_t nRow = 0; + TSDBROW row = {.version = version}; + + SMemSkipListNode *backward[SL_MAX_LEVEL]; + SMemSkipListNode *forward[SL_MAX_LEVEL]; + + ASSERT(pSubmitBlk->nData); + + // backward put first data + n += tGetTSRow(p + n, &row.tsRow); + ASSERT(n <= pSubmitBlk->nData); + + memDataGetPosToPut(pMemData, backward, &row); + code = memDataDoPut(pMemData, backward, &row, 0); + if (code) { + goto _exit; + } + nRow++; + + // forward put rest + while (n < pSubmitBlk->nData) { + n += tGetTSRow(p + n, &row.tsRow); + ASSERT(n <= pSubmitBlk->nData); - if (isForward) { // TODO - } else { - SMemSkipListNode *px = pMemData->sl.pTail; + nRow++; + } + +_exit: + return code; +} + +static void memDataGetPosToPut(SMemData *pMemData, SMemSkipListNode **backward, TSDBROW *pRow) { + SMemSkipListNode *px; + SMemSkipListNode *pn; + TSDBKEY *pKey; + int c; + + for (int8_t iLevel = 0; iLevel < pMemData->sl.maxLevel - 1; iLevel++) { + backward[iLevel] = pMemData->sl.pTail; + } + + if (0) { + } else { + px = pMemData->sl.pTail; for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= 0; iLevel--) { if (iLevel < pMemData->sl.level) { - SMemSkipListNode *p = SL_NODE_BACKWARD(px, iLevel); - - while (p != pMemData->sl.pHead) { - pKey = (TSDBKEY *)SL_NODE_DATA(p); + pn = SL_NODE_BACKWARD(px, iLevel); + while (pn != pMemData->sl.pHead) { + pKey = (TSDBKEY *)SL_NODE_DATA(pn); c = tsdbKeyCmprFn(pKey, pRow); if (c <= 0) { break; } else { - px = p; - p = SL_NODE_BACKWARD(px, iLevel); + px = pn; + pn = SL_NODE_BACKWARD(px, iLevel); } } - - pos[iLevel] = px; } + backward[iLevel] = px; } } } -static void memMovePosFrom(SMemData *pMemData, SMemSkipListNode *pNode, TSDBROW *pRow, int8_t isForward, - SMemSkipListNode **pos) { - SMemSkipListNode *px = pNode; - TSDBKEY *pKey; - SMemSkipListNode *p; - int c; - - if (isForward) { - } else { - ASSERT(pNode != pMemData->sl.pHead); - - for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= 0; iLevel--) { - p = SL_NODE_BACKWARD(px, iLevel); - while (p != pMemData->sl.pHead) { - pKey = (TSDBKEY *)SL_NODE_DATA(p); - - c = tsdbKeyCmprFn(pKey, pRow); - if (c <= 0) { - break; - } else { - px = p; - p = SL_NODE_BACKWARD(px, iLevel); - } - } - - pos[iLevel] = px; - } - } -} - -static int32_t memDataPutRow(SVBufPool *pPool, SMemData *pMemData, TSDBROW *pRow, int8_t isForward, - SMemSkipListNode **pos) { - int32_t code = 0; - int8_t level; - SMemSkipListNode *pNode; - - level = tsdbMemSkipListRandLevel(&pMemData->sl); - pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + tPutTSDBRow(NULL, pRow)); - if (pNode == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - // do the read put - if (isForward) { - // TODO - } else { - // TODO - } - -_exit: +static int32_t memDataDoPut(SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow, int8_t isForward) { + int32_t code = 0; + // TODO return code; } \ No newline at end of file From c937313e3f5b7e54ea6bba17222c5d0024c35fb1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 2 Jun 2022 22:31:12 +0800 Subject: [PATCH 02/63] feat:add json test case --- source/client/src/clientImpl.c | 2 +- source/common/src/tmsg.c | 2 +- tests/system-test/2-query/json_tag.py | 92 +++++++++++++-------------- 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index e7b4fb0657..1ae528d78c 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1175,7 +1175,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); varDataSetLen(dst, strlen(varDataVal(dst))); } else if (jsonInnerType == TD_TAG_JSON) { - char* jsonString = parseTagDatatoJson(jsonInnerData); + char* jsonString = parseTagDatatoJson(pStart); STR_TO_VARSTR(dst, jsonString); taosMemoryFree(jsonString); } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 9c6c532bcd..ed32d5a16b 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -34,7 +34,7 @@ int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { return -1; } - pIter->totalLen = htonl(pMsg->length); + pIter->totalLen = ntohl(pMsg->length); ASSERT(pIter->totalLen > 0); pIter->len = 0; pIter->pMsg = pMsg; diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index d8ef8fa363..486abf9f40 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -48,8 +48,8 @@ class TDTestCase: # test duplicate key using the first one. elimate empty key tdSql.execute("CREATE TABLE if not exists jsons1_8 using jsons1 tags('{\"tag1\":null, \"tag1\":true, \"tag1\":45, \"1tag$\":2, \" \":90, \"\":32}')") - #tdSql.query("select jtag from jsons1_8") - #tdSql.checkData(0, 0, '{"tag1":null,"1tag$":2," ":90}') + tdSql.query("select jtag from jsons1_8") + tdSql.checkData(0, 0, '{" ":90,"1tag$":2,"tag1":null}') # test empty json string, save as jtag is NULL tdSql.execute("insert into jsons1_9 using jsons1 tags('\t') values (1591062328000, 24, NULL, '你就会', '2sdw')") @@ -95,8 +95,8 @@ class TDTestCase: tdSql.error("ALTER TABLE jsons1 MODIFY TAG jtag nchar(128)") # tdSql.execute("ALTER TABLE jsons1_1 SET TAG jtag='{\"tag1\":\"femail\",\"tag2\":35,\"tag3\":true}'") - # tdSql.query("select jtag from jsons1_1") - # tdSql.checkData(0, 0, '{"tag1":"femail","tag2":35,"tag3":true}') + tdSql.query("select jtag from jsons1_1") + tdSql.checkData(0, 0, '{"tag1":"femail","tag2":35,"tag3":true}') tdSql.execute("ALTER TABLE jsons1 rename TAG jtag jtag_new") tdSql.execute("ALTER TABLE jsons1 rename TAG jtag_new jtag") @@ -106,18 +106,18 @@ class TDTestCase: # # print("============== STEP 3 ===== query table") # # test error syntax - # tdSql.error("select * from jsons1 where jtag->tag1='beijing'") - # tdSql.error("select * from jsons1 where jtag->'location'") - # tdSql.error("select * from jsons1 where jtag->''") - # tdSql.error("select * from jsons1 where jtag->''=9") - # tdSql.error("select -> from jsons1") - # tdSql.error("select * from jsons1 where contains") - # tdSql.error("select * from jsons1 where jtag->") - # tdSql.error("select jtag->location from jsons1") - # tdSql.error("select jtag contains location from jsons1") - # tdSql.error("select * from jsons1 where jtag contains location") - # tdSql.error("select * from jsons1 where jtag contains''") - # tdSql.error("select * from jsons1 where jtag contains 'location'='beijing'") + tdSql.error("select * from jsons1 where jtag->tag1='beijing'") + #tdSql.error("select * from jsons1 where jtag->'location'") + #tdSql.error("select * from jsons1 where jtag->''") + #tdSql.error("select * from jsons1 where jtag->''=9") + tdSql.error("select -> from jsons1") + tdSql.error("select * from jsons1 where contains") + tdSql.error("select * from jsons1 where jtag->") + tdSql.error("select jtag->location from jsons1") + tdSql.error("select jtag contains location from jsons1") + tdSql.error("select * from jsons1 where jtag contains location") + tdSql.error("select * from jsons1 where jtag contains''") + tdSql.error("select * from jsons1 where jtag contains 'location'='beijing'") # # # test function error # tdSql.error("select avg(jtag->'tag1') from jsons1") @@ -128,47 +128,47 @@ class TDTestCase: # tdSql.error("select ceil(jtag) from jsons1") # # # test select normal column - # tdSql.query("select dataint from jsons1") - # tdSql.checkRows(9) - # tdSql.checkData(1, 0, 1) + tdSql.query("select dataint from jsons1") + tdSql.checkRows(9) + tdSql.checkData(1, 0, 1) # test select json tag - # tdSql.query("select * from jsons1") - # tdSql.checkRows(8) - # tdSql.query("select jtag from jsons1") - # tdSql.checkRows(7) + tdSql.query("select * from jsons1") + tdSql.checkRows(9) + tdSql.query("select jtag from jsons1") + tdSql.checkRows(13) # tdSql.query("select jtag from jsons1 where jtag is null") # tdSql.checkRows(5) # tdSql.query("select jtag from jsons1 where jtag is not null") # tdSql.checkRows(8) # test jtag is NULL - #tdSql.query("select jtag from jsons1_9") - #tdSql.checkData(0, 0, None) + tdSql.query("select jtag from jsons1_9") + tdSql.checkData(0, 0, None) # # test select json tag->'key', value is string - # tdSql.query("select jtag->'tag1' from jsons1_1") - # tdSql.checkData(0, 0, '"femail"') - # tdSql.query("select jtag->'tag2' from jsons1_6") - # tdSql.checkData(0, 0, '""') - # # test select json tag->'key', value is int - # tdSql.query("select jtag->'tag2' from jsons1_1") - # tdSql.checkData(0, 0, 35) - # # test select json tag->'key', value is bool - # tdSql.query("select jtag->'tag3' from jsons1_1") - # tdSql.checkData(0, 0, "true") - # # test select json tag->'key', value is null - # tdSql.query("select jtag->'tag1' from jsons1_4") - # tdSql.checkData(0, 0, "null") - # # test select json tag->'key', value is double - # tdSql.query("select jtag->'tag1' from jsons1_5") - # tdSql.checkData(0, 0, "1.232000000") - # # test select json tag->'key', key is not exist - # tdSql.query("select jtag->'tag10' from jsons1_4") - # tdSql.checkData(0, 0, None) + tdSql.query("select jtag->'tag1' from jsons1_1") + tdSql.checkData(0, 0, '"femail"') + tdSql.query("select jtag->'tag2' from jsons1_6") + tdSql.checkData(0, 0, '""') + # test select json tag->'key', value is int + tdSql.query("select jtag->'tag2' from jsons1_1") + tdSql.checkData(0, 0, "35.000000000") + # test select json tag->'key', value is bool + tdSql.query("select jtag->'tag3' from jsons1_1") + tdSql.checkData(0, 0, "true") + # test select json tag->'key', value is null + tdSql.query("select jtag->'tag1' from jsons1_4") + tdSql.checkData(0, 0, "null") + # test select json tag->'key', value is double + tdSql.query("select jtag->'tag1' from jsons1_5") + tdSql.checkData(0, 0, "1.232000000") + # test select json tag->'key', key is not exist + tdSql.query("select jtag->'tag10' from jsons1_4") + tdSql.checkData(0, 0, None) # - # tdSql.query("select jtag->'tag1' from jsons1") - # tdSql.checkRows(13) + tdSql.query("select jtag->'tag1' from jsons1") + tdSql.checkRows(13) # test header name res = tdSql.getColNameList("select jtag->'tag1' from jsons1") cname_list = [] From 802f6d246c93ae5110b9390474840a235ffe83be Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 4 Jun 2022 01:26:05 +0000 Subject: [PATCH 03/63] more --- include/common/tdataformat.h | 3 +++ source/common/src/tdataformat.c | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 073d796717..6a73d3d678 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -41,6 +41,9 @@ typedef struct STag STag; int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); void tTSchemaDestroy(STSchema *pTSchema); +// SValue +int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type); + // STSRow2 #define COL_VAL_NONE(CID) ((SColVal){.cid = (CID), .isNone = 1}) #define COL_VAL_NULL(CID) ((SColVal){.cid = (CID), .isNull = 1}) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 287dba6d3b..9aa94b0216 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -36,8 +36,6 @@ typedef struct { #define GET_BIT1(p, i) (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1)) #define GET_BIT2(p, i) (((p)[(i) / 4] >> ((i) % 4)) & ((uint8_t)3)) -static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2); - // SValue static FORCE_INLINE int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { int32_t n = 0; @@ -141,6 +139,11 @@ static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { return n; } +int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type) { + // TODO + return 0; +} + // STSRow2 ======================================================================== static void setBitMap(uint8_t *pb, uint8_t v, int32_t idx, uint8_t flags) { if (pb) { From c568aeaa4374e3421d153b4aea5b23de38aaa1ef Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 4 Jun 2022 03:41:13 +0000 Subject: [PATCH 04/63] more --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 42 ++++++++++++++------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 3bad5fdd92..d32caed77b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -64,8 +64,8 @@ static int memDataPCmprFn(const void *p1, const void *p2); static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow); static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl); -static void memDataGetPosToPut(SMemData *pMemData, SMemSkipListNode **backward, TSDBROW *pRow); -static int32_t memDataDoPut(SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow, int8_t isForward); +static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow, int8_t forward); +static int32_t memDataDoPut(SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow, int8_t forward); static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, int64_t version, SVSubmitBlk *pSubmitBlk); @@ -300,8 +300,7 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, int32_t nRow = 0; TSDBROW row = {.version = version}; - SMemSkipListNode *backward[SL_MAX_LEVEL]; - SMemSkipListNode *forward[SL_MAX_LEVEL]; + SMemSkipListNode *pos[SL_MAX_LEVEL]; ASSERT(pSubmitBlk->nData); @@ -309,8 +308,8 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, n += tGetTSRow(p + n, &row.tsRow); ASSERT(n <= pSubmitBlk->nData); - memDataGetPosToPut(pMemData, backward, &row); - code = memDataDoPut(pMemData, backward, &row, 0); + memDataMovePosTo(pMemData, pos, &row, 0); + code = memDataDoPut(pMemData, pos, &row, 0); if (code) { goto _exit; } @@ -322,6 +321,11 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, ASSERT(n <= pSubmitBlk->nData); // TODO + memDataMovePosTo(pMemData, pos, &row, 1); + code = memDataDoPut(pMemData, pos, &row, 1); + if (code) { + goto _exit; + } nRow++; } @@ -330,17 +334,29 @@ _exit: return code; } -static void memDataGetPosToPut(SMemData *pMemData, SMemSkipListNode **backward, TSDBROW *pRow) { +static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow, int8_t forward) { SMemSkipListNode *px; SMemSkipListNode *pn; TSDBKEY *pKey; int c; - for (int8_t iLevel = 0; iLevel < pMemData->sl.maxLevel - 1; iLevel++) { - backward[iLevel] = pMemData->sl.pTail; - } + if (forward) { + px = pMemData->sl.pHead; + for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= 0; iLevel--) { + pn = SL_NODE_FORWARD(px, iLevel); + while (pn != pMemData->sl.pTail) { + pKey = (TSDBKEY *)SL_NODE_DATA(pn); - if (0) { + c = tsdbKeyCmprFn(pKey, pRow); + if (pKey >= 0) { + break; + } else { + px = pn; + pn = SL_NODE_FORWARD(px, iLevel); + } + } + pos[iLevel] = px; + } } else { px = pMemData->sl.pTail; for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= 0; iLevel--) { @@ -358,12 +374,12 @@ static void memDataGetPosToPut(SMemData *pMemData, SMemSkipListNode **backward, } } } - backward[iLevel] = px; + pos[iLevel] = px; } } } -static int32_t memDataDoPut(SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow, int8_t isForward) { +static int32_t memDataDoPut(SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow, int8_t forward) { int32_t code = 0; // TODO return code; From 9fe400f20719e7bfa9dc9892c06d6a13fc4f7342 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 4 Jun 2022 03:52:26 +0000 Subject: [PATCH 05/63] more --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 55 +++++++++++---------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index d32caed77b..78be665ae9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -59,12 +59,15 @@ struct SMemTable { #define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) #define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level)) +#define SL_MOVE_BACKWARD 0x1 +#define SL_MOVE_FROM_POS 0x2 + static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData); static int memDataPCmprFn(const void *p1, const void *p2); static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow); static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl); -static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow, int8_t forward); +static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags); static int32_t memDataDoPut(SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow, int8_t forward); static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, int64_t version, SVSubmitBlk *pSubmitBlk); @@ -308,7 +311,7 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, n += tGetTSRow(p + n, &row.tsRow); ASSERT(n <= pSubmitBlk->nData); - memDataMovePosTo(pMemData, pos, &row, 0); + memDataMovePosTo(pMemData, pos, &(TSDBKEY){.version = version, .ts = row.tsRow.ts}, SL_MOVE_BACKWARD); code = memDataDoPut(pMemData, pos, &row, 0); if (code) { goto _exit; @@ -321,7 +324,7 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, ASSERT(n <= pSubmitBlk->nData); // TODO - memDataMovePosTo(pMemData, pos, &row, 1); + memDataMovePosTo(pMemData, pos, &(TSDBKEY){.version = version, .ts = row.tsRow.ts}, SL_MOVE_FROM_POS); code = memDataDoPut(pMemData, pos, &row, 1); if (code) { goto _exit; @@ -334,38 +337,23 @@ _exit: return code; } -static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow, int8_t forward) { +static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags) { SMemSkipListNode *px; SMemSkipListNode *pn; - TSDBKEY *pKey; + TSDBKEY *pTKey; int c; + int backward = flags & SL_MOVE_BACKWARD; + int fromPos = flags & SL_MOVE_FROM_POS; - if (forward) { - px = pMemData->sl.pHead; - for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= 0; iLevel--) { - pn = SL_NODE_FORWARD(px, iLevel); - while (pn != pMemData->sl.pTail) { - pKey = (TSDBKEY *)SL_NODE_DATA(pn); - - c = tsdbKeyCmprFn(pKey, pRow); - if (pKey >= 0) { - break; - } else { - px = pn; - pn = SL_NODE_FORWARD(px, iLevel); - } - } - pos[iLevel] = px; - } - } else { + if (backward) { px = pMemData->sl.pTail; for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= 0; iLevel--) { if (iLevel < pMemData->sl.level) { pn = SL_NODE_BACKWARD(px, iLevel); while (pn != pMemData->sl.pHead) { - pKey = (TSDBKEY *)SL_NODE_DATA(pn); + pTKey = (TSDBKEY *)SL_NODE_DATA(pn); - c = tsdbKeyCmprFn(pKey, pRow); + c = tsdbKeyCmprFn(pTKey, pKey); if (c <= 0) { break; } else { @@ -376,6 +364,23 @@ static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBROW } pos[iLevel] = px; } + } else { + px = pMemData->sl.pHead; + for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= 0; iLevel--) { + pn = SL_NODE_FORWARD(px, iLevel); + while (pn != pMemData->sl.pTail) { + pTKey = (TSDBKEY *)SL_NODE_DATA(pn); + + c = tsdbKeyCmprFn(pTKey, pKey); + if (pTKey >= 0) { + break; + } else { + px = pn; + pn = SL_NODE_FORWARD(px, iLevel); + } + } + pos[iLevel] = px; + } } } From 2b022f665c01d330f30d6a33fb3708c467e72084 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 4 Jun 2022 04:22:00 +0000 Subject: [PATCH 06/63] more --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 48 ++++++++++++++------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 78be665ae9..3b2d97092d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -347,8 +347,15 @@ static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY if (backward) { px = pMemData->sl.pTail; - for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= 0; iLevel--) { - if (iLevel < pMemData->sl.level) { + + for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= pMemData->sl.level; iLevel--) { + pos[iLevel] = px; + } + + if (pMemData->sl.level) { + if (fromPos) px = pos[pMemData->sl.level - 1]; + + for (int8_t iLevel = pMemData->sl.level - 1; iLevel >= 0; iLevel--) { pn = SL_NODE_BACKWARD(px, iLevel); while (pn != pMemData->sl.pHead) { pTKey = (TSDBKEY *)SL_NODE_DATA(pn); @@ -361,26 +368,37 @@ static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY pn = SL_NODE_BACKWARD(px, iLevel); } } + + pos[iLevel] = px; } - pos[iLevel] = px; } } else { px = pMemData->sl.pHead; - for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= 0; iLevel--) { - pn = SL_NODE_FORWARD(px, iLevel); - while (pn != pMemData->sl.pTail) { - pTKey = (TSDBKEY *)SL_NODE_DATA(pn); - c = tsdbKeyCmprFn(pTKey, pKey); - if (pTKey >= 0) { - break; - } else { - px = pn; - pn = SL_NODE_FORWARD(px, iLevel); - } - } + for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= pMemData->sl.level; iLevel--) { pos[iLevel] = px; } + + if (pMemData->sl.level) { + if (fromPos) px = pos[pMemData->sl.level - 1]; + + for (int8_t iLevel = pMemData->sl.level - 1; iLevel >= 0; iLevel--) { + pn = SL_NODE_FORWARD(px, iLevel); + while (pn != pMemData->sl.pHead) { + pTKey = (TSDBKEY *)SL_NODE_DATA(pn); + + c = tsdbKeyCmprFn(pTKey, pKey); + if (c >= 0) { + break; + } else { + px = pn; + pn = SL_NODE_FORWARD(px, iLevel); + } + } + + pos[iLevel] = px; + } + } } } From 41a1b0fcf2f1046619552262479fca574a923d42 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 4 Jun 2022 12:42:50 +0800 Subject: [PATCH 07/63] refactor: add null transaction action typ --- include/common/tmsgdef.h | 3 +- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 1 - source/dnode/mnode/impl/inc/mndTrans.h | 11 ++- source/dnode/mnode/impl/src/mndMnode.c | 27 ++----- source/dnode/mnode/impl/src/mndTrans.c | 83 ++++++++++++++------- 5 files changed, 73 insertions(+), 52 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 2cf188866b..012bc40c8a 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -152,8 +152,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_SYSTABLE_RETRIEVE, "retrieve", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "grant", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "auth", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CONFIRM_WRITE, "mnode-confirm-write", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_APPLY_MSG, "mnode-apply-msg", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_APPLY_MSG, "mnode-apply", NULL, NULL) TD_NEW_MSG_SEG(TDMT_VND_MSG) TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 4a79513736..78ab850964 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -197,7 +197,6 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIRM_WRITE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 6d1f371083..736fa3b361 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -29,12 +29,18 @@ typedef enum { TRANS_STOP_FUNC_MQ_REB = 4, } ETrnFunc; +typedef enum { + TRANS_ACTION_NULL = 0, + TRANS_ACTION_MSG = 1, + TRANS_ACTION_RAW = 2, +} ETrnAct; + typedef struct { int32_t id; int32_t errCode; int32_t acceptableCode; int8_t stage; - int8_t actionType; // 0-msg, 1-raw + ETrnAct actionType; int8_t rawWritten; int8_t msgSent; int8_t msgReceived; @@ -57,6 +63,7 @@ void mndTransDrop(STrans *pTrans); int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw); +int32_t mndTransAppendNullLog(STrans *pTrans); int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction); int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction); void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); @@ -65,7 +72,7 @@ void mndTransSetDbName(STrans *pTrans, const char *dbname); void mndTransSetSerial(STrans *pTrans); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); -void mndTransProcessRsp(SRpcMsg *pRsp); +int32_t mndTransProcessRsp(SRpcMsg *pRsp); void mndTransPullup(SMnode *pMnode); int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 013a991e98..1cc832b3af 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -34,9 +34,6 @@ static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOld, SMnodeObj *pNe static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq); static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq); static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq); -static int32_t mndProcessCreateMnodeRsp(SRpcMsg *pRsp); -static int32_t mndProcessAlterMnodeRsp(SRpcMsg *pRsp); -static int32_t mndProcessDropMnodeRsp(SRpcMsg *pRsp); static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter); @@ -53,11 +50,11 @@ int32_t mndInitMnode(SMnode *pMnode) { }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_MNODE, mndProcessCreateMnodeReq); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_MNODE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_ALTER_MNODE, mndProcessAlterMnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_ALTER_MNODE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_DROP_MNODE, mndProcessDropMnodeReq); - mndSetMsgHandle(pMnode, TDMT_DND_CREATE_MNODE_RSP, mndProcessCreateMnodeRsp); - mndSetMsgHandle(pMnode, TDMT_MND_ALTER_MNODE_RSP, mndProcessAlterMnodeRsp); - mndSetMsgHandle(pMnode, TDMT_DND_DROP_MNODE_RSP, mndProcessDropMnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_MNODE_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_MNODE, mndRetrieveMnodes); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_MNODE, mndCancelGetNextMnode); @@ -367,7 +364,7 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; - + if (mndTransAppendNullLog(pTrans) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -549,6 +546,7 @@ static int32_t mndDropMnode(SMnode *pMnode, SRpcMsg *pReq, SMnodeObj *pObj) { if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) goto _OVER; if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) goto _OVER; if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) goto _OVER; + if (mndTransAppendNullLog(pTrans) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -616,21 +614,6 @@ _OVER: return code; } -static int32_t mndProcessCreateMnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - -static int32_t mndProcessAlterMnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - -static int32_t mndProcessDropMnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index bbee59090d..cc9cc6edc2 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -88,12 +88,14 @@ static int32_t mndTransGetActionsSize(SArray *pArray) { for (int32_t i = 0; i < actionNum; ++i) { STransAction *pAction = taosArrayGet(pArray, i); - if (pAction->actionType) { + if (pAction->actionType == TRANS_ACTION_RAW) { rawDataLen += (sdbGetRawTotalSize(pAction->pRaw) + sizeof(int32_t)); - } else { + } else if (pAction->actionType == TRANS_ACTION_MSG) { rawDataLen += (sizeof(STransAction) + pAction->contLen); + } else { + // empty } - rawDataLen += sizeof(pAction->actionType); + rawDataLen += sizeof(int8_t); } return rawDataLen; @@ -137,18 +139,20 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) - if (pAction->actionType) { + if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) SDB_SET_INT32(pRaw, dataPos, len, _OVER) SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pRaw, len, _OVER) - } else { + } else if (pAction->actionType == TRANS_ACTION_MSG) { SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->msgSent, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->msgReceived, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) + } else { + // nothing } } @@ -159,18 +163,20 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) - if (pAction->actionType) { + if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) SDB_SET_INT32(pRaw, dataPos, len, _OVER) SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pRaw, len, _OVER) - } else { + } else if (pAction->actionType == TRANS_ACTION_MSG) { SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->msgSent, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->msgReceived, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) + } else { + // nothing } } @@ -181,18 +187,20 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) - if (pAction->actionType) { + if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) SDB_SET_INT32(pRaw, dataPos, len, _OVER) SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pRaw, len, _OVER) - } else { + } else if (pAction->actionType == TRANS_ACTION_MSG) { SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->msgSent, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->msgReceived, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) + } else { + // nothing } } @@ -252,6 +260,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { int16_t policy = 0; int16_t conflict = 0; int16_t exec = 0; + int8_t actionType = 0; SDB_GET_INT16(pRaw, dataPos, &stage, _OVER) SDB_GET_INT16(pRaw, dataPos, &policy, _OVER) SDB_GET_INT16(pRaw, dataPos, &conflict, _OVER) @@ -279,9 +288,10 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) - SDB_GET_INT8(pRaw, dataPos, &action.actionType, _OVER) + SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) + action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &action.stage, _OVER) - if (action.actionType) { + if (action.actionType == TRANS_ACTION_RAW) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) action.pRaw = taosMemoryMalloc(dataLen); @@ -290,7 +300,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; action.pRaw = NULL; - } else { + } else if (action.actionType == TRANS_ACTION_MSG) { SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) SDB_GET_INT8(pRaw, dataPos, &action.msgSent, _OVER) @@ -301,6 +311,8 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; action.pCont = NULL; + } else { + if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; } } @@ -308,9 +320,10 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) - SDB_GET_INT8(pRaw, dataPos, &action.actionType, _OVER) + SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) + action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &action.stage, _OVER) - if (action.actionType) { + if (action.actionType == TRANS_ACTION_RAW) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) action.pRaw = taosMemoryMalloc(dataLen); @@ -319,7 +332,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; action.pRaw = NULL; - } else { + } else if (action.actionType == TRANS_ACTION_MSG) { SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) SDB_GET_INT8(pRaw, dataPos, &action.msgSent, _OVER) @@ -330,6 +343,8 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; action.pCont = NULL; + } else { + if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; } } @@ -337,7 +352,8 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) - SDB_GET_INT8(pRaw, dataPos, &action.actionType, _OVER) + SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) + action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &action.stage, _OVER) if (action.actionType) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) @@ -348,7 +364,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); if (taosArrayPush(pTrans->commitActions, &action) == NULL) goto _OVER; action.pRaw = NULL; - } else { + } else if (action.actionType == TRANS_ACTION_MSG) { SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) SDB_GET_INT8(pRaw, dataPos, &action.msgSent, _OVER) @@ -359,6 +375,8 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); if (taosArrayPush(pTrans->commitActions, &action) == NULL) goto _OVER; action.pCont = NULL; + } else { + if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; } } @@ -552,10 +570,12 @@ static void mndTransDropActions(SArray *pArray) { int32_t size = taosArrayGetSize(pArray); for (int32_t i = 0; i < size; ++i) { STransAction *pAction = taosArrayGet(pArray, i); - if (pAction->actionType) { + if (pAction->actionType == TRANS_ACTION_RAW) { taosMemoryFreeClear(pAction->pRaw); - } else { + } else if (pAction->actionType == TRANS_ACTION_MSG) { taosMemoryFreeClear(pAction->pCont); + } else { + // nothing } } @@ -583,27 +603,34 @@ static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction) { } int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw) { - STransAction action = {.stage = TRN_STAGE_REDO_ACTION, .actionType = true, .pRaw = pRaw}; + STransAction action = {.stage = TRN_STAGE_REDO_ACTION, .actionType = TRANS_ACTION_RAW, .pRaw = pRaw}; + return mndTransAppendAction(pTrans->redoActions, &action); +} + +int32_t mndTransAppendNullLog(STrans *pTrans) { + STransAction action = {.stage = TRN_STAGE_REDO_ACTION, .actionType = TRANS_ACTION_NULL}; return mndTransAppendAction(pTrans->redoActions, &action); } int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw) { - STransAction action = {.stage = TRN_STAGE_UNDO_ACTION, .actionType = true, .pRaw = pRaw}; + STransAction action = {.stage = TRN_STAGE_UNDO_ACTION, .actionType = TRANS_ACTION_RAW, .pRaw = pRaw}; return mndTransAppendAction(pTrans->undoActions, &action); } int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) { - STransAction action = {.stage = TRN_STAGE_COMMIT_ACTION, .actionType = true, .pRaw = pRaw}; + STransAction action = {.stage = TRN_STAGE_COMMIT_ACTION, .actionType = TRANS_ACTION_RAW, .pRaw = pRaw}; return mndTransAppendAction(pTrans->commitActions, &action); } int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction) { pAction->stage = TRN_STAGE_REDO_ACTION; + pAction->actionType = TRANS_ACTION_MSG; return mndTransAppendAction(pTrans->redoActions, pAction); } int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction) { pAction->stage = TRN_STAGE_UNDO_ACTION; + pAction->actionType = TRANS_ACTION_MSG; return mndTransAppendAction(pTrans->undoActions, pAction); } @@ -782,7 +809,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { } } -void mndTransProcessRsp(SRpcMsg *pRsp) { +int32_t mndTransProcessRsp(SRpcMsg *pRsp) { SMnode *pMnode = pRsp->info.node; int64_t signature = (int64_t)(pRsp->info.ahandle); int32_t transId = (int32_t)(signature >> 32); @@ -827,6 +854,7 @@ void mndTransProcessRsp(SRpcMsg *pRsp) { _OVER: mndReleaseTrans(pMnode, pTrans); + return 0; } static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { @@ -899,10 +927,15 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio } static int32_t mndTransExecSingleAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { - if (pAction->actionType) { + if (pAction->actionType == TRANS_ACTION_RAW) { return mndTransWriteSingleLog(pMnode, pTrans, pAction); - } else { + } else if (pAction->actionType == TRANS_ACTION_MSG) { return mndTransSendSingleMsg(pMnode, pTrans, pAction); + } else { + pAction->rawWritten = 0; + pAction->errCode = 0; + mDebug("trans:%d, %s:%d null action executed", pTrans->id, mndTransStr(pAction->stage), pAction->id); + return 0; } } From a0aa003f89a6497b0f4d44d5e5db2940d745ec3c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 4 Jun 2022 06:29:01 +0000 Subject: [PATCH 08/63] more --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 126 ++++++++++++-------- 1 file changed, 78 insertions(+), 48 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 3b2d97092d..d067130a12 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -67,8 +67,6 @@ static int memDataPCmprFn(const void *p1, const void *p2); static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow); static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl); -static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags); -static int32_t memDataDoPut(SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow, int8_t forward); static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, int64_t version, SVSubmitBlk *pSubmitBlk); @@ -295,48 +293,6 @@ static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { return level; } -static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, int64_t version, - SVSubmitBlk *pSubmitBlk) { - int32_t code = 0; - int32_t n = 0; - uint8_t *p = pSubmitBlk->pData; - int32_t nRow = 0; - TSDBROW row = {.version = version}; - - SMemSkipListNode *pos[SL_MAX_LEVEL]; - - ASSERT(pSubmitBlk->nData); - - // backward put first data - n += tGetTSRow(p + n, &row.tsRow); - ASSERT(n <= pSubmitBlk->nData); - - memDataMovePosTo(pMemData, pos, &(TSDBKEY){.version = version, .ts = row.tsRow.ts}, SL_MOVE_BACKWARD); - code = memDataDoPut(pMemData, pos, &row, 0); - if (code) { - goto _exit; - } - nRow++; - - // forward put rest - while (n < pSubmitBlk->nData) { - n += tGetTSRow(p + n, &row.tsRow); - ASSERT(n <= pSubmitBlk->nData); - - // TODO - memDataMovePosTo(pMemData, pos, &(TSDBKEY){.version = version, .ts = row.tsRow.ts}, SL_MOVE_FROM_POS); - code = memDataDoPut(pMemData, pos, &row, 1); - if (code) { - goto _exit; - } - - nRow++; - } - -_exit: - return code; -} - static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags) { SMemSkipListNode *px; SMemSkipListNode *pn; @@ -402,8 +358,82 @@ static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY } } -static int32_t memDataDoPut(SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow, int8_t forward) { - int32_t code = 0; - // TODO +static int32_t memDataDoPut(SMemTable *pMemTable, SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow, + int8_t forward) { + int32_t code = 0; + int8_t level; + SMemSkipListNode *pNode; + SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; + + // node + level = tsdbMemSkipListRandLevel(&pMemData->sl); + pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + tPutTSDBRow(NULL, pRow)); + if (pNode == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + pNode->level = level; + for (int8_t iLevel = 0; iLevel < level; iLevel++) { + SL_NODE_FORWARD(pNode, iLevel) = NULL; + SL_NODE_BACKWARD(pNode, iLevel) = NULL; + } + + tPutTSDBRow((uint8_t *)SL_NODE_DATA(pNode), pRow); + + // put + for (int8_t iLevel = 0; iLevel < pNode->level; iLevel++) { + /* code */ + } + + pMemData->sl.size++; + if (pMemData->sl.level < pNode->level) { + pMemData->sl.level = pNode->level; + } + +_exit: return code; -} \ No newline at end of file +} + +static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, int64_t version, + SVSubmitBlk *pSubmitBlk) { + int32_t code = 0; + int32_t n = 0; + uint8_t *p = pSubmitBlk->pData; + int32_t nRow = 0; + TSDBROW row = {.version = version}; + + SMemSkipListNode *pos[SL_MAX_LEVEL]; + + ASSERT(pSubmitBlk->nData); + + // backward put first data + n += tGetTSRow(p + n, &row.tsRow); + ASSERT(n <= pSubmitBlk->nData); + + memDataMovePosTo(pMemData, pos, &(TSDBKEY){.version = version, .ts = row.tsRow.ts}, SL_MOVE_BACKWARD); + code = memDataDoPut(pMemTable, pMemData, pos, &row, 0); + if (code) { + goto _exit; + } + nRow++; + + // forward put rest + for (int8_t iLevel = 0; iLevel < pMemData->sl.maxLevel; iLevel++) { + pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel); + } + while (n < pSubmitBlk->nData) { + n += tGetTSRow(p + n, &row.tsRow); + ASSERT(n <= pSubmitBlk->nData); + + memDataMovePosTo(pMemData, pos, &(TSDBKEY){.version = version, .ts = row.tsRow.ts}, SL_MOVE_FROM_POS); + code = memDataDoPut(pMemTable, pMemData, pos, &row, 1); + if (code) { + goto _exit; + } + + nRow++; + } + +_exit: + return code; +} From f439918d51281cb372a610e2c58599452a0848d6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 4 Jun 2022 06:46:22 +0000 Subject: [PATCH 09/63] more --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index d067130a12..866bd69cda 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -382,7 +382,25 @@ static int32_t memDataDoPut(SMemTable *pMemTable, SMemData *pMemData, SMemSkipLi // put for (int8_t iLevel = 0; iLevel < pNode->level; iLevel++) { - /* code */ + SMemSkipListNode *px = pos[iLevel]; + + if (forward) { + SMemSkipListNode *pNext = SL_NODE_FORWARD(px, iLevel); + + SL_NODE_FORWARD(pNode, iLevel) = pNext; + SL_NODE_BACKWARD(pNode, iLevel) = px; + + SL_NODE_BACKWARD(pNext, iLevel) = pNode; + SL_NODE_FORWARD(px, iLevel) = pNode; + } else { + SMemSkipListNode *pPrev = SL_NODE_BACKWARD(px, iLevel); + + SL_NODE_FORWARD(pNode, iLevel) = px; + SL_NODE_BACKWARD(pNode, iLevel) = pPrev; + + SL_NODE_FORWARD(pPrev, iLevel) = pNode; + SL_NODE_BACKWARD(px, iLevel) = pNode; + } } pMemData->sl.size++; From 23494b8ff10f07288c992512ccd1489808b80701 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 4 Jun 2022 06:57:10 +0000 Subject: [PATCH 10/63] more --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 866bd69cda..b6d0747fe3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -435,6 +435,13 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, } nRow++; + if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemData->minKey) < 0) { + pMemData->minKey = *(TSDBKEY *)&row; + } + if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemTable->minKey) < 0) { + pMemTable->minKey = *(TSDBKEY *)&row; + } + // forward put rest for (int8_t iLevel = 0; iLevel < pMemData->sl.maxLevel; iLevel++) { pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel); @@ -452,6 +459,14 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, nRow++; } + if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemData->maxKey) > 0) { + pMemData->maxKey = *(TSDBKEY *)&row; + } + if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemTable->maxKey) > 0) { + pMemTable->maxKey = *(TSDBKEY *)&row; + } + pMemTable->nRows += nRow; + _exit: return code; } From c7c47788f35fb67c46c94065d378b1cf370f5f80 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 4 Jun 2022 07:38:27 +0000 Subject: [PATCH 11/63] more --- source/dnode/vnode/src/inc/vnodeInt.h | 4 +-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 40 +++++++++++++----------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index d3b5f29aac..c4da909c21 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -87,7 +87,7 @@ int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* p int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq); int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids); -int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp *pMetaRsp); +int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp); SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); int metaGetTableEntryByName(SMetaReader* pReader, const char* name); @@ -112,7 +112,7 @@ int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg); int tsdbClose(STsdb** pTsdb); int tsdbBegin(STsdb* pTsdb); -int tsdbCommit(STsdb* pTsdb); +int32_t tsdbCommit(STsdb* pTsdb); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 9e5a5a751b..928915a36d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -86,7 +86,7 @@ static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError); static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo); static void tsdbLoadAndMergeFromCache(STsdb *pTsdb, SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, TSKEY maxKey, int maxRows, int8_t update); -int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf); +static int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf); static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn); int tsdbBegin(STsdb *pTsdb) { @@ -101,33 +101,36 @@ int tsdbBegin(STsdb *pTsdb) { return 0; } -int tsdbCommit(STsdb *pRepo) { +int32_t tsdbCommit(STsdb *pTsdb) { + int32_t code = 0; SCommitH commith = {0}; SDFileSet *pSet = NULL; int fid; - // if (pRepo->imem == NULL) return 0; - pRepo->imem = pRepo->mem; - pRepo->mem = NULL; + ASSERT(pTsdb->imem == NULL && pTsdb->mem); + pTsdb->imem = pTsdb->mem; + pTsdb->mem = NULL; - tsdbStartCommit(pRepo); - // Resource initialization - if (tsdbInitCommitH(&commith, pRepo) < 0) { + // start commit + tsdbStartCommit(pTsdb); + if (tsdbInitCommitH(&commith, pTsdb) < 0) { return -1; } +#if 0 // Skip expired memory data and expired FSET tsdbSeekCommitIter(&commith, commith.rtn.minKey); while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) { if (pSet->fid < commith.rtn.minFid) { - tsdbInfo("vgId:%d, FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, + tsdbInfo("vgId:%d, FSET %d on level %d disk id %d expires, remove it", REPO_ID(pTsdb), pSet->fid, TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); } else { break; } } +#endif - // Loop to commit to each file + // commit fid = tsdbNextCommitFid(&(commith)); while (true) { // Loop over both on disk and memory @@ -136,7 +139,7 @@ int tsdbCommit(STsdb *pRepo) { if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) { // Only has existing FSET but no memory data to commit in this // existing FSET, only check if file in correct retention - if (tsdbApplyRtnOnFSet(pRepo, pSet, &(commith.rtn)) < 0) { + if (tsdbApplyRtnOnFSet(pTsdb, pSet, &(commith.rtn)) < 0) { tsdbDestroyCommitH(&commith); return -1; } @@ -167,10 +170,11 @@ int tsdbCommit(STsdb *pRepo) { } } + // end commit tsdbDestroyCommitH(&commith); - tsdbEndCommit(pRepo, TSDB_CODE_SUCCESS); + tsdbEndCommit(pTsdb, TSDB_CODE_SUCCESS); - return 0; + return code; } static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) { @@ -717,8 +721,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid // extern int32_t tsTsdbMetaCompactRatio; -int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf, - SBlockIdx *pIdx) { +static int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf, + SBlockIdx *pIdx) { size_t nSupBlocks; size_t nSubBlocks; uint32_t tlen; @@ -778,7 +782,7 @@ int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray return 0; } -int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { +static int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { SBlockIdx *pBlkIdx; size_t nidx = taosArrayGetSize(pIdxA); int tlen = 0, size; @@ -1012,8 +1016,8 @@ static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { * @param ppExBuf * @return int */ -int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols, - SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf) { +static int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols, + SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf) { STsdbCfg *pCfg = REPO_CFG(pRepo); SBlockData *pBlockData = NULL; SAggrBlkData *pAggrBlkData = NULL; From 8f16b9b8c11eca9e463cb4fbcaf5c48a4a8fcfb7 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sat, 4 Jun 2022 15:59:41 +0800 Subject: [PATCH 12/63] catalog force update --- include/libs/catalog/catalog.h | 3 +- include/libs/qcom/query.h | 2 +- source/client/src/clientEnv.c | 8 +-- source/client/src/clientImpl.c | 14 +++++ source/client/src/clientMsgHandler.c | 2 +- source/libs/catalog/inc/catalogInt.h | 21 +++++-- source/libs/catalog/src/ctgAsync.c | 6 ++ source/libs/catalog/src/ctgCache.c | 80 ++++++++++++++++++++++++--- source/libs/catalog/src/ctgDbg.c | 3 +- source/libs/scheduler/src/schJob.c | 25 +++++++-- source/libs/scheduler/src/schRemote.c | 1 + 11 files changed, 136 insertions(+), 29 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index f0e642bc9a..ee237741c3 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -68,6 +68,7 @@ typedef struct SCatalogReq { SArray *pIndex; // element is index name SArray *pUser; // element is SUserAuthInfo bool qNodeRequired; // valid qnode + bool forceUpdate; } SCatalogReq; typedef struct SMetaData { @@ -280,7 +281,7 @@ int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet *epSet); -int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, uint64_t reqId); +int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, uint64_t reqId, bool forceUpdate); /** diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 45a7e9a29f..a17be44846 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -222,7 +222,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t || (_type) == TDMT_VND_DROP_TABLE || (_type) == TDMT_VND_DROP_STB) #define NEED_SCHEDULER_RETRY_ERROR(_code) \ - ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) + ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) #define REQUEST_MAX_TRY_TIMES 1 diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 4b39a51584..9afb989d27 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -223,6 +223,10 @@ static void doDestroyRequest(void *p) { taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self)); + if (pRequest->body.queryJob != 0) { + schedulerFreeJob(pRequest->body.queryJob); + } + taosMemoryFreeClear(pRequest->msgBuf); taosMemoryFreeClear(pRequest->sqlstr); taosMemoryFreeClear(pRequest->pDb); @@ -230,10 +234,6 @@ static void doDestroyRequest(void *p) { doFreeReqResultInfo(&pRequest->body.resInfo); qDestroyQueryPlan(pRequest->body.pDag); - if (pRequest->body.queryJob != 0) { - schedulerFreeJob(pRequest->body.queryJob); - } - taosArrayDestroy(pRequest->tableList); taosArrayDestroy(pRequest->dbList); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 774ef5f248..9762ac6ba5 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -741,6 +741,20 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { do { destroyRequest(pRequest); pRequest = launchQuery(pTscObj, sql, sqlLen); + if (*sql == 'y') { + SCatalog *pCatalog = NULL; + code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); + SEpSet epset = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + ctgdLaunchAsyncCall(pCatalog, pTscObj->pAppInfo->pTransporter, &epset, pRequest->requestId, false); + break; + } else if (*sql == 'z') { + SCatalog *pCatalog = NULL; + code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); + SEpSet epset = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + ctgdLaunchAsyncCall(pCatalog, pTscObj->pAppInfo->pTransporter, &epset, pRequest->requestId, false); + break; + } + if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) { break; } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 1039d36362..2d5199f181 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -180,7 +180,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { taosMemoryFreeClear(output.dbVgroup); tscError("0x%" PRIx64 " failed to build use db output since %s", pRequest->requestId, terrstr()); - } else if (output.dbVgroup) { + } else if (output.dbVgroup && output.dbVgroup->vgHash) { struct SCatalog* pCatalog = NULL; int32_t code1 = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index cebe696390..9219a382e4 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -52,6 +52,7 @@ enum { CTG_OP_UPDATE_VGROUP = 0, CTG_OP_UPDATE_TB_META, CTG_OP_DROP_DB_CACHE, + CTG_OP_DROP_DB_VGROUP, CTG_OP_DROP_STB_META, CTG_OP_DROP_TB_META, CTG_OP_UPDATE_USER, @@ -266,26 +267,32 @@ typedef struct SCtgUpdateTblMsg { STableMetaOutput* output; } SCtgUpdateTblMsg; -typedef struct SCtgRemoveDBMsg { +typedef struct SCtgDropDBMsg { SCatalog* pCtg; char dbFName[TSDB_DB_FNAME_LEN]; uint64_t dbId; -} SCtgRemoveDBMsg; +} SCtgDropDBMsg; -typedef struct SCtgRemoveStbMsg { +typedef struct SCtgDropDbVgroupMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; +} SCtgDropDbVgroupMsg; + + +typedef struct SCtgDropStbMetaMsg { SCatalog* pCtg; char dbFName[TSDB_DB_FNAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN]; uint64_t dbId; uint64_t suid; -} SCtgRemoveStbMsg; +} SCtgDropStbMetaMsg; -typedef struct SCtgRemoveTblMsg { +typedef struct SCtgDropTblMetaMsg { SCatalog* pCtg; char dbFName[TSDB_DB_FNAME_LEN]; char tbName[TSDB_TABLE_NAME_LEN]; uint64_t dbId; -} SCtgRemoveTblMsg; +} SCtgDropTblMetaMsg; typedef struct SCtgUpdateUserMsg { SCatalog* pCtg; @@ -451,6 +458,7 @@ int32_t ctgGetTbMetaFromCache(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMeta** pTabl int32_t ctgOpUpdateVgroup(SCtgCacheOperation *action); int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *action); int32_t ctgOpDropDbCache(SCtgCacheOperation *action); +int32_t ctgOpDropDbVgroup(SCtgCacheOperation *action); int32_t ctgOpDropStbMeta(SCtgCacheOperation *action); int32_t ctgOpDropTbMeta(SCtgCacheOperation *action); int32_t ctgOpUpdateUser(SCtgCacheOperation *action); @@ -464,6 +472,7 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName); int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass); int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId); +int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char *dbFName, bool syncReq); int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq); int32_t ctgDropTbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq); int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 312f0c9250..47ef45d7d7 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -286,6 +286,9 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* int32_t taskIdx = 0; for (int32_t i = 0; i < dbVgNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbVgroup, i); + if (pReq->forceUpdate) { + ctgDropDbVgroupEnqueue(pCtg, dbFName, true); + } CTG_ERR_JRET(ctgInitGetDbVgTask(pJob, taskIdx++, dbFName)); } @@ -301,6 +304,9 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* for (int32_t i = 0; i < tbMetaNum; ++i) { SName* name = taosArrayGet(pReq->pTableMeta, i); + if (pReq->forceUpdate) { + catalogRemoveTableMeta(pCtg, name); + } CTG_ERR_JRET(ctgInitGetTbMetaTask(pJob, taskIdx++, name)); } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 0f1344c343..8332c7b068 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -35,6 +35,11 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = { "drop DB", ctgOpDropDbCache }, + { + CTG_OP_DROP_DB_VGROUP, + "drop DBVgroup", + ctgOpDropDbVgroup + }, { CTG_OP_DROP_STB_META, "drop stbMeta", @@ -563,9 +568,9 @@ int32_t ctgEnqueue(SCatalog* pCtg, SCtgCacheOperation *operation) { int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) { int32_t code = 0; SCtgCacheOperation action= {.opId = CTG_OP_DROP_DB_CACHE}; - SCtgRemoveDBMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveDBMsg)); + SCtgDropDBMsg *msg = taosMemoryMalloc(sizeof(SCtgDropDBMsg)); if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); + ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropDBMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -590,13 +595,43 @@ _return: CTG_RET(code); } +int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char *dbFName, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation action= {.opId = CTG_OP_DROP_DB_VGROUP, .syncOp = syncOp}; + SCtgDropDbVgroupMsg *msg = taosMemoryMalloc(sizeof(SCtgDropDbVgroupMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropDbVgroupMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + char *p = strchr(dbFName, '.'); + if (p && CTG_IS_SYS_DBNAME(p + 1)) { + dbFName = p + 1; + } + + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + + action.data = msg; + + CTG_ERR_JRET(ctgEnqueue(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(action.data); + CTG_RET(code); +} + + int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncOp) { int32_t code = 0; SCtgCacheOperation action= {.opId = CTG_OP_DROP_STB_META, .syncOp = syncOp}; - SCtgRemoveStbMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveStbMsg)); + SCtgDropStbMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgDropStbMetaMsg)); if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); + ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropStbMetaMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -623,9 +658,9 @@ _return: int32_t ctgDropTbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncOp) { int32_t code = 0; SCtgCacheOperation action= {.opId = CTG_OP_DROP_TB_META, .syncOp = syncOp}; - SCtgRemoveTblMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveTblMsg)); + SCtgDropTblMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgDropTblMetaMsg)); if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveTblMsg)); + ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTblMetaMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1281,7 +1316,7 @@ _return: int32_t ctgOpDropDbCache(SCtgCacheOperation *operation) { int32_t code = 0; - SCtgRemoveDBMsg *msg = operation->data; + SCtgDropDBMsg *msg = operation->data; SCatalog* pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; @@ -1304,6 +1339,33 @@ _return: CTG_RET(code); } +int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgDropDbVgroupMsg *msg = operation->data; + SCatalog* pCtg = msg->pCtg; + + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + goto _return; + } + + CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); + + ctgFreeVgInfo(dbCache->vgInfo); + dbCache->vgInfo = NULL; + + ctgDebug("db vgInfo removed, dbFName:%s", msg->dbFName); + + ctgWReleaseVgInfo(dbCache); + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) { int32_t code = 0; @@ -1353,7 +1415,7 @@ _return: int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { int32_t code = 0; - SCtgRemoveStbMsg *msg = operation->data; + SCtgDropStbMetaMsg *msg = operation->data; SCatalog* pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; @@ -1399,7 +1461,7 @@ _return: int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { int32_t code = 0; - SCtgRemoveTblMsg *msg = operation->data; + SCtgDropTblMetaMsg *msg = operation->data; SCatalog* pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index fdab50db0f..cb18e45638 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -132,7 +132,7 @@ void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { } } -int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, uint64_t reqId) { +int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, uint64_t reqId, bool forceUpdate) { int32_t code = 0; SCatalogReq req = {0}; req.pTableMeta = taosArrayInit(2, sizeof(SName)); @@ -144,6 +144,7 @@ int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps req.pIndex = NULL;//taosArrayInit(2, TSDB_INDEX_FNAME_LEN); req.pUser = taosArrayInit(2, sizeof(SUserAuthInfo)); req.qNodeRequired = true; + req.forceUpdate = forceUpdate; SName name = {0}; char dbFName[TSDB_DB_FNAME_LEN] = {0}; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 3e23c395c9..be7b930d79 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -377,15 +377,21 @@ int32_t schDropTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int32_ return TSDB_CODE_SUCCESS; } - taosHashRemove(pTask->execNodes, &execIdx, sizeof(execIdx)); + if (taosHashRemove(pTask->execNodes, &execIdx, sizeof(execIdx))) { + SCH_TASK_ELOG("fail to remove execIdx %d from execNodeList", execIdx); + } else { + SCH_TASK_DLOG("execIdx %d removed from execNodeList", execIdx); + } + if (execIdx != pTask->execIdx) { // ignore it + SCH_TASK_DLOG("execIdx %d is not current execIdx %d", execIdx, pTask->execIdx); SCH_RET(TSDB_CODE_SCH_IGNORE_ERROR); } return TSDB_CODE_SUCCESS; } -int32_t schUpdateTaskExecNode(SSchTask *pTask, void *handle, int32_t execIdx) { +int32_t schUpdateTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int32_t execIdx) { if (taosHashGetSize(pTask->execNodes) <= 0) { return TSDB_CODE_SUCCESS; } @@ -393,6 +399,8 @@ int32_t schUpdateTaskExecNode(SSchTask *pTask, void *handle, int32_t execIdx) { SSchNodeInfo *nodeInfo = taosHashGet(pTask->execNodes, &execIdx, sizeof(execIdx)); nodeInfo->handle = handle; + SCH_TASK_DLOG("handle updated to %p for execIdx %d", handle, execIdx); + return TSDB_CODE_SUCCESS; } @@ -403,7 +411,7 @@ int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, v SCH_SET_TASK_HANDLE(pTask, handle); - schUpdateTaskExecNode(pTask, handle, execIdx); + schUpdateTaskExecNode(pJob, pTask, handle, execIdx); return TSDB_CODE_SUCCESS; } @@ -551,6 +559,8 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + SCH_TASK_DLOG("set %dth condidate addr, id %d, fqdn:%s, port:%d", i, naddr->nodeId, SCH_GET_CUR_EP(naddr)->fqdn, SCH_GET_CUR_EP(naddr)->port); + ++addNum; } } @@ -1110,6 +1120,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_UNLOCK(SCH_WRITE, &parent->lock); if (SCH_TASK_READY_FOR_LAUNCH(readyNum, parent)) { + SCH_TASK_DLOG("all %d children task done, start to launch parent task %" PRIx64, readyNum, parent->taskId); SCH_ERR_RET(schLaunchTask(pJob, parent)); } } @@ -1186,7 +1197,7 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) { nodeInfo = taosHashIterate(pTask->execNodes, nodeInfo); } - SCH_TASK_DLOG("task has %d exec address", size); + SCH_TASK_DLOG("task has been dropped on %d exec nodes", size); } @@ -1196,7 +1207,8 @@ int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) { } SCH_LOCK_TASK(pTask); - if (JOB_TASK_STATUS_EXECUTING == pTask->status && pJob->fetchTask != pTask) { + if (JOB_TASK_STATUS_EXECUTING == pTask->status && pJob->fetchTask != pTask && taosArrayGetSize(pTask->candidateAddrs) > 1) { + SCH_TASK_DLOG("task execIdx %d will be rescheduled now", pTask->execIdx); schDropTaskOnExecNode(pJob, pTask); taosHashClear(pTask->execNodes); schProcessOnTaskFailure(pJob, pTask, TSDB_CODE_SCH_TIMEOUT_ERROR); @@ -1306,9 +1318,10 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { int32_t code = 0; atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); - pTask->execIdx++; + SCH_TASK_DLOG("start to launch task's %dth exec", pTask->execIdx); + SCH_LOG_TASK_START_TS(pTask); if (schJobNeedToStop(pJob, &status)) { diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index b336ce8c76..1389459604 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1037,6 +1037,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); isCandidateAddr = true; + SCH_TASK_DLOG("target candidateIdx %d", pTask->candidateIdx); } SEpSet epSet = addr->epSet; From de32cab384d8eec3131e0230bfdbf3e618ed32e2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 4 Jun 2022 08:27:55 +0000 Subject: [PATCH 13/63] more progress --- source/dnode/vnode/CMakeLists.txt | 1 + source/dnode/vnode/src/inc/tsdb.h | 14 +++ source/dnode/vnode/src/tsdb/tsdbCommit2.c | 104 ++++++++++++++++++++ source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 24 ++--- 4 files changed, 129 insertions(+), 14 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/tsdbCommit2.c diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index f3aefdba02..05b4a270d6 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -36,6 +36,7 @@ target_sources( # tsdb "src/tsdb/tsdbCommit.c" + "src/tsdb/tsdbCommit2.c" "src/tsdb/tsdbFile.c" "src/tsdb/tsdbFS.c" "src/tsdb/tsdbOpen.c" diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ab23b7506e..c9b8a26735 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -46,6 +46,10 @@ void tsdbMemTableDestroy2(SMemTable *pMemTable); int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk); int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); +// tsdbCommit2.c ============================================================================================== +int32_t tsdbBegin2(STsdb *pTsdb); +int32_t tsdbCommit2(STsdb *pTsdb); + // tsdbMemTable ================ typedef struct STsdbRow STsdbRow; typedef struct STbData STbData; @@ -877,6 +881,16 @@ struct SDelOp { SDelOp *pNext; }; +struct SMemTable { + STsdb *pTsdb; + int32_t nRef; + TSDBKEY minKey; + TSDBKEY maxKey; + int64_t nRows; + SArray *aSkmInfo; + SArray *aMemData; +}; + static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) { TSDBKEY *pKey1 = (TSDBKEY *)p1; TSDBKEY *pKey2 = (TSDBKEY *)p2; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c new file mode 100644 index 0000000000..3e87e4be43 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +typedef struct { + SMemTable *pMemTable; + SArray *aBlkIdx; +} SCommitH; + +static int32_t tsdbStartCommit(SCommitH *pCHandle, STsdb *pTsdb); +static int32_t tsdbEndCommit(SCommitH *pCHandle); +static int32_t tsdbCommitToFile(SCommitH *pCHandle, int32_t fid); + +int32_t tsdbBegin2(STsdb *pTsdb) { + int32_t code = 0; + + ASSERT(pTsdb->mem == NULL); + code = tsdbMemTableCreate2(pTsdb, (SMemTable **)&pTsdb->mem); + if (code) { + tsdbError("vgId:%d failed to begin TSDB since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + goto _exit; + } + +_exit: + return code; +} + +int32_t tsdbCommit2(STsdb *pTsdb) { + int32_t code = 0; + SCommitH ch = {0}; + + // start to commit + code = tsdbStartCommit(&ch, pTsdb); + if (code) { + goto _exit; + } + + // commit + int32_t sfid; // todo + int32_t efid; // todo + for (int32_t fid = sfid; fid <= efid; fid++) { + code = tsdbCommitToFile(&ch, fid); + if (code) { + goto _err; + } + } + + // end commit + code = tsdbEndCommit(&ch); + if (code) { + goto _exit; + } + +_exit: + return code; + +_err: + // TODO: rollback + return code; +} + +static int32_t tsdbStartCommit(SCommitH *pCHandle, STsdb *pTsdb) { + int32_t code = 0; + + ASSERT(pTsdb->imem == NULL && pTsdb->mem); + pTsdb->imem = pTsdb->mem; + pTsdb->mem = NULL; + + return code; +} + +static int32_t tsdbEndCommit(SCommitH *pCHandle) { + int32_t code = 0; + // TODO + return code; +} + +static int32_t tsdbCommitToFile(SCommitH *pCHandle, int32_t fid) { + int32_t code = 0; + TSKEY fidSKey; + TSKEY fidEKey; + + // check if there are data in the time range + for (int32_t iMemData = 0; iMemData < taosArrayGetSize(pCHandle->pMemTable->aMemData); iMemData++) { + /* code */ + } + + // has data, do commit to file + + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index b6d0747fe3..23f779ec88 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -43,14 +43,10 @@ struct SMemData { SMemSkipList sl; }; -struct SMemTable { - STsdb *pTsdb; - int32_t nRef; - TSDBKEY minKey; - TSDBKEY maxKey; - int64_t nRows; - SArray *pArray; // SArray -}; +typedef struct { + tb_uid_t uid; + STSchema *pTSchema; +} SSkmInfo; #define SL_MAX_LEVEL 5 @@ -85,8 +81,8 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) { pMemTable->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX}; pMemTable->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN}; pMemTable->nRows = 0; - pMemTable->pArray = taosArrayInit(512, sizeof(SMemData *)); - if (pMemTable->pArray == NULL) { + pMemTable->aMemData = taosArrayInit(512, sizeof(SMemData *)); + if (pMemTable->aMemData == NULL) { taosMemoryFree(pMemTable); code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -101,7 +97,7 @@ _err: } void tsdbMemTableDestroy2(SMemTable *pMemTable) { - taosArrayDestroyEx(pMemTable->pArray, NULL /*TODO*/); + taosArrayDestroyEx(pMemTable->aMemData, NULL /*TODO*/); taosMemoryFree(pMemTable); } @@ -196,9 +192,9 @@ static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_ui int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel; // get - idx = taosArraySearchIdx(pMemTable->pArray, &pMemDataT, memDataPCmprFn, TD_GE); + idx = taosArraySearchIdx(pMemTable->aMemData, &pMemDataT, memDataPCmprFn, TD_GE); if (idx >= 0) { - pMemData = (SMemData *)taosArrayGet(pMemTable->pArray, idx); + pMemData = (SMemData *)taosArrayGet(pMemTable->aMemData, idx); if (memDataPCmprFn(&pMemDataT, &pMemData) == 0) goto _exit; } @@ -230,7 +226,7 @@ static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_ui } if (idx < 0) idx = 0; - if (taosArrayInsert(pMemTable->pArray, idx, &pMemData) == NULL) { + if (taosArrayInsert(pMemTable->aMemData, idx, &pMemData) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } From ad6a648d2b3ecb53e5b1debccf890ed6277e8d7f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 4 Jun 2022 17:08:46 +0800 Subject: [PATCH 14/63] refactor: do some internal refactor. --- source/client/inc/clientInt.h | 3 +-- source/libs/catalog/src/ctgAsync.c | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index c7b831893e..09116a6bd1 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -45,8 +45,7 @@ extern "C" { #define ERROR_MSG_BUF_DEFAULT_SIZE 512 #define HEARTBEAT_INTERVAL 1500 // ms - -#define SYNC_ON_TOP_OF_ASYNC 0 +#define SYNC_ON_TOP_OF_ASYNC 0 enum { RES_TYPE__QUERY = 1, diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index a9e808e749..ce94eace66 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -44,7 +44,7 @@ int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, task.type, name->tname); + qDebug("QID:0x%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, task.type, name->tname); return TSDB_CODE_SUCCESS; } @@ -143,7 +143,7 @@ int32_t ctgInitGetTbHashTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, task.type, name->tname); + qDebug("QID:0x%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, task.type, name->tname); return TSDB_CODE_SUCCESS; } @@ -247,7 +247,7 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* *taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dbCfgNum + indexNum + userNum + dbInfoNum; if (*taskNum <= 0) { - ctgError("Empty input for job, no need to retrieve meta, reqId:0x%" PRIx64, reqId); + ctgDebug("Empty input for job, no need to retrieve meta, reqId:0x%" PRIx64, reqId); return TSDB_CODE_SUCCESS; } From 0bc5300d03a569480ff92fd271f4b4f31ec5f1ef Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 4 Jun 2022 09:31:35 +0000 Subject: [PATCH 15/63] more progress --- source/dnode/vnode/src/inc/tsdb.h | 16 +++++++- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 43 +++++++++++++++++---- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 1 - 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index c9b8a26735..3a2eec30ba 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -39,13 +39,21 @@ typedef struct SDelOp SDelOp; static int tsdbKeyCmprFn(const void *p1, const void *p2); // tsdbMemTable2.c ============================================================================================== -typedef struct SMemTable SMemTable; +typedef struct SMemTable SMemTable; +typedef struct SMemData SMemData; +typedef struct SMemDataIter SMemDataIter; int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable); void tsdbMemTableDestroy2(SMemTable *pMemTable); int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk); int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); +/* SMemDataIter */ +void tsdbMemDataIterOpen(SMemDataIter *pIter, TSDBKEY *pKey, int8_t backward); +void tsdbMemDataIterClose(SMemDataIter *pIter); +void tsdbMemDataIterNext(SMemDataIter *pIter); +void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow); + // tsdbCommit2.c ============================================================================================== int32_t tsdbBegin2(STsdb *pTsdb); int32_t tsdbCommit2(STsdb *pTsdb); @@ -254,6 +262,7 @@ typedef struct { uint32_t offset; uint32_t hasLast : 2; uint32_t numOfBlocks : 30; + uint64_t suid; uint64_t uid; TSKEY maxKey; } SBlockIdx; @@ -910,6 +919,11 @@ static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) { return 0; } +struct SMemDataIter { + SMemData *pMemData; + TSDBROW row; +}; + #endif #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 3e87e4be43..6bb53a1263 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -78,7 +78,7 @@ static int32_t tsdbStartCommit(SCommitH *pCHandle, STsdb *pTsdb) { ASSERT(pTsdb->imem == NULL && pTsdb->mem); pTsdb->imem = pTsdb->mem; pTsdb->mem = NULL; - + // TODO return code; } @@ -89,16 +89,45 @@ static int32_t tsdbEndCommit(SCommitH *pCHandle) { } static int32_t tsdbCommitToFile(SCommitH *pCHandle, int32_t fid) { - int32_t code = 0; - TSKEY fidSKey; - TSKEY fidEKey; + int32_t code = 0; + SMemDataIter iter = {0}; + TSDBROW *pRow = NULL; + int8_t hasData = 0; + TSKEY fidSKey; + TSKEY fidEKey; + int32_t iMemData = 0; + int32_t nMemData = taosArrayGetSize(pCHandle->pMemTable->aMemData); + int32_t iBlockIdx = 0; + int32_t nBlockIdx; // check if there are data in the time range - for (int32_t iMemData = 0; iMemData < taosArrayGetSize(pCHandle->pMemTable->aMemData); iMemData++) { - /* code */ + for (; iMemData < nMemData; iMemData++) { + SMemData *pMemData = (SMemData *)taosArrayGetP(pCHandle->pMemTable->aMemData, iMemData); + tsdbMemDataIterOpen(&iter, &(TSDBKEY){.ts = fidSKey, .version = 0}, 0); + tsdbMemDataIterGet(&iter, &pRow); + + if (pRow->tsRow.ts >= fidSKey && pRow->tsRow.ts <= fidEKey) { + hasData = 1; + break; + } } - // has data, do commit to file + if (!hasData) return code; + + // loop to commit each table data + nBlockIdx = 0; + for (;;) { + if (iBlockIdx >= nBlockIdx && iMemData >= nMemData) break; + + SMemData *pMemData = NULL; + SBlockIdx *pBlockIdx = NULL; + if (iMemData < nMemData) { + pMemData = (SMemData *)taosArrayGetP(pCHandle->pMemTable->aMemData, iBlockIdx); + } + if (iBlockIdx < nBlockIdx) { + // pBlockIdx + } + } return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 23f779ec88..1c28b086b3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,7 +15,6 @@ #include "tsdb.h" -typedef struct SMemData SMemData; typedef struct SMemSkipList SMemSkipList; typedef struct SMemSkipListNode SMemSkipListNode; From a2ebeda89c2ec4309baac4c9d04edd5c8a568b3b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 4 Jun 2022 19:28:30 +0800 Subject: [PATCH 16/63] feat: add sort/group logic for json --- include/common/tdatablock.h | 2 ++ source/common/src/tdatablock.c | 32 ++++++++++++--------- source/libs/executor/src/groupoperator.c | 36 +++++++++++++++++++++--- source/libs/index/src/indexFilter.c | 16 ++++------- source/util/src/tcompare.c | 29 +++++++++++++++++++ tests/system-test/2-query/json_tag.py | 21 +++++++------- 6 files changed, 97 insertions(+), 39 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 66b81efc5b..cce763cafe 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -186,6 +186,8 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u *(double*)p = *(double*)v; } +int32_t getJsonValueLen(const char *data); + int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, uint32_t numOfRow2); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index f77b823f3c..055992db53 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -99,6 +99,24 @@ void colDataTrim(SColumnInfoData* pColumnInfoData) { // TODO } +int32_t getJsonValueLen(const char *data) { + int32_t dataLen = 0; + if (*data == TSDB_DATA_TYPE_NULL) { + dataLen = CHAR_BYTES; + } else if (*data == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(data + CHAR_BYTES) + CHAR_BYTES; + } else if (*data == TSDB_DATA_TYPE_DOUBLE) { + dataLen = DOUBLE_BYTES + CHAR_BYTES; + } else if (*data == TSDB_DATA_TYPE_BOOL) { + dataLen = CHAR_BYTES + CHAR_BYTES; + } else if (*data == TD_TAG_JSON) { // json string + dataLen = ((STag*)(data))->len; + } else { + ASSERT(0); + } + return dataLen; +} + int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull) { ASSERT(pColumnInfoData != NULL); @@ -118,19 +136,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con if (IS_VAR_DATA_TYPE(type)) { int32_t dataLen = 0; if (type == TSDB_DATA_TYPE_JSON) { - if (*pData == TSDB_DATA_TYPE_NULL) { - dataLen = CHAR_BYTES; - } else if (*pData == TSDB_DATA_TYPE_NCHAR) { - dataLen = varDataTLen(pData + CHAR_BYTES) + CHAR_BYTES; - } else if (*pData == TSDB_DATA_TYPE_DOUBLE) { - dataLen = DOUBLE_BYTES + CHAR_BYTES; - } else if (*pData == TSDB_DATA_TYPE_BOOL) { - dataLen = CHAR_BYTES + CHAR_BYTES; - } else if (*pData == TD_TAG_JSON) { // json string - dataLen = ((STag*)(pData))->len; - } else { - ASSERT(0); - } + dataLen = getJsonValueLen(pData); }else { dataLen = varDataTLen(pData); } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 3691024096..0a6b3149ef 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -93,7 +93,15 @@ static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlo char* val = colDataGetData(pColInfoData, rowIndex); - if (IS_VAR_DATA_TYPE(pkey->type)) { + if (pkey->type == TSDB_DATA_TYPE_JSON) { + int32_t dataLen = getJsonValueLen(val); + + if (memcmp(pkey->pData, val, dataLen) == 0){ + continue; + } else { + return false; + } + } else if (IS_VAR_DATA_TYPE(pkey->type)) { int32_t len = varDataLen(val); if (len == varDataLen(pkey->pData) && memcmp(varDataVal(pkey->pData), varDataVal(val), len) == 0) { continue; @@ -129,7 +137,10 @@ void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* } else { pkey->isNull = false; char* val = colDataGetData(pColInfoData, rowIndex); - if (IS_VAR_DATA_TYPE(pkey->type)) { + if (pkey->type == TSDB_DATA_TYPE_JSON) { + int32_t dataLen = getJsonValueLen(val); + memcpy(pkey->pData, val, dataLen); + } else if (IS_VAR_DATA_TYPE(pkey->type)) { memcpy(pkey->pData, val, varDataTLen(val)); ASSERT(varDataTLen(val) <= pkey->bytes); } else { @@ -153,7 +164,11 @@ int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals) { } isNull[i] = 0; - if (IS_VAR_DATA_TYPE(pkey->type)) { + if (pkey->type == TSDB_DATA_TYPE_JSON) { + int32_t dataLen = getJsonValueLen(pkey->pData); + memcpy(pStart, (pkey->pData), dataLen); + pStart += dataLen; + } else if (IS_VAR_DATA_TYPE(pkey->type)) { varDataCopy(pStart, pkey->pData); pStart += varDataTLen(pkey->pData); ASSERT(varDataTLen(pkey->pData) <= pkey->bytes); @@ -178,7 +193,10 @@ static void doAssignGroupKeys(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t char* dest = GET_ROWCELL_INTERBUF(pEntryInfo); char* data = colDataGetData(pColInfoData, rowIndex); - if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { + int32_t dataLen = getJsonValueLen(data); + memcpy(dest, data, dataLen); + } else if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { varDataCopy(dest, data); } else { memcpy(dest, data, pColInfoData->info.bytes); @@ -447,6 +465,16 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { if (colDataIsNull_s(pColInfoData, j)) { offset[(*rows)] = -1; contentLen = 0; + } else if(pColInfoData->info.type == TSDB_DATA_TYPE_JSON){ + offset[*rows] = (*columnLen); + char* src = colDataGetData(pColInfoData, j); + int32_t dataLen = getJsonValueLen(src); + + memcpy(data + (*columnLen), src, dataLen); + int32_t v = (data + (*columnLen) + dataLen - (char*)pPage); + ASSERT(v > 0); + + contentLen = dataLen; } else { offset[*rows] = (*columnLen); char* src = colDataGetData(pColInfoData, j); diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index b41006b6dd..9f5fa4cb27 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -140,7 +140,7 @@ static int32_t sifGetValueFromNode(SNode *node, char **value) { dataLen = 0; } else if (*pData == TSDB_DATA_TYPE_NCHAR) { dataLen = varDataTLen(pData + CHAR_BYTES); - } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { + } else if (*pData == TSDB_DATA_TYPE_DOUBLE) { dataLen = LONG_BYTES; } else if (*pData == TSDB_DATA_TYPE_BOOL) { dataLen = CHAR_BYTES; @@ -457,10 +457,6 @@ static int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *sta static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { int32_t code = 0; int32_t nParam = sifGetOperParamNum(node->opType); - if (nParam <= 1) { - SIF_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - SIFParam *params = NULL; SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); @@ -469,14 +465,12 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { sif_func_t operFn = sifNullFunc; code = sifGetOperFn(node->opType, &operFn, &output->status); - if (ctx->noExec) { - SIF_RET(code); - } else { - return operFn(¶ms[0], nParam > 1 ? ¶ms[1] : NULL, output); + if (!ctx->noExec) { + code = operFn(¶ms[0], nParam > 1 ? ¶ms[1] : NULL, output); } -_return: + taosMemoryFree(params); - SIF_RET(code); + return code; } static int32_t sifExecLogic(SLogicConditionNode *node, SIFCtx *ctx, SIFParam *output) { diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 7f1fdf3306..b660927899 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -227,6 +227,33 @@ int32_t compareJsonContainsKey(const void* pLeft, const void* pRight) { return 1; } +// string > number > bool > null +// ref: https://dev.mysql.com/doc/refman/8.0/en/json.html#json-comparison +int32_t compareJsonVal(const void *pLeft, const void *pRight) { + char leftType = *(char*)pLeft; + char rightType = *(char*)pRight; + if(leftType != rightType){ + return leftType > rightType ? 1 : -1; + } + + char* realDataLeft = POINTER_SHIFT(pLeft, CHAR_BYTES); + char* realDataRight = POINTER_SHIFT(pRight, CHAR_BYTES); + if(leftType == TSDB_DATA_TYPE_BOOL) { + DEFAULT_COMP(GET_INT8_VAL(realDataLeft), GET_INT8_VAL(realDataRight)); + }else if(leftType == TSDB_DATA_TYPE_DOUBLE){ + DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(realDataLeft), GET_DOUBLE_VAL(realDataRight)); + }else if(leftType == TSDB_DATA_TYPE_NCHAR){ + return compareLenPrefixedWStr(realDataLeft, realDataRight); + }else if(leftType == TSDB_DATA_TYPE_NULL) { + return 0; + }else{ + assert(0); + } +} + +int32_t compareJsonValDesc(const void *pLeft, const void *pRight) { + return compareJsonVal(pRight, pLeft); +} /* * Compare two strings * TSDB_MATCH: Match @@ -601,6 +628,8 @@ __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { return (order == TSDB_ORDER_ASC) ? compareLenPrefixedStr : compareLenPrefixedStrDesc; case TSDB_DATA_TYPE_NCHAR: return (order == TSDB_ORDER_ASC) ? compareLenPrefixedWStr : compareLenPrefixedWStrDesc; + case TSDB_DATA_TYPE_JSON: + return (order == TSDB_ORDER_ASC) ? compareJsonVal : compareJsonValDesc; default: return (order == TSDB_ORDER_ASC) ? compareInt32Val : compareInt32ValDesc; } diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 486abf9f40..f6a32cbe95 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -120,12 +120,12 @@ class TDTestCase: tdSql.error("select * from jsons1 where jtag contains 'location'='beijing'") # # # test function error - # tdSql.error("select avg(jtag->'tag1') from jsons1") - # tdSql.error("select avg(jtag) from jsons1") - # tdSql.error("select min(jtag->'tag1') from jsons1") - # tdSql.error("select min(jtag) from jsons1") - # tdSql.error("select ceil(jtag->'tag1') from jsons1") - # tdSql.error("select ceil(jtag) from jsons1") + tdSql.error("select avg(jtag->'tag1') from jsons1") + tdSql.error("select avg(jtag) from jsons1") + tdSql.error("select min(jtag->'tag1') from jsons1") + tdSql.error("select min(jtag) from jsons1") + tdSql.error("select ceil(jtag->'tag1') from jsons1") + tdSql.error("select ceil(jtag) from jsons1") # # # test select normal column tdSql.query("select dataint from jsons1") @@ -176,7 +176,6 @@ class TDTestCase: tdSql.checkColNameList(res, cname_list) - # # test where with json tag # tdSql.error("select * from jsons1_1 where jtag is not null") # tdSql.error("select * from jsons1 where jtag='{\"tag1\":11,\"tag2\":\"\"}'") @@ -313,8 +312,8 @@ class TDTestCase: # tdSql.checkRows(2) # # # test with tbname/normal column - # tdSql.query("select * from jsons1 where tbname = 'jsons1_1'") - # tdSql.checkRows(2) + tdSql.query("select * from jsons1 where tbname = 'jsons1_1'") + tdSql.checkRows(2) # tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag contains 'tag3'") # tdSql.checkRows(2) # tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag contains 'tag3' and dataint=3") @@ -345,14 +344,14 @@ class TDTestCase: # tdSql.checkRows(1) # # # test distinct - # tdSql.execute("insert into jsons1_14 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":null}') values(1591062628000, 2, NULL, '你就会', 'dws')") + tdSql.execute("insert into jsons1_14 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":null}') values(1591062628000, 2, NULL, '你就会', 'dws')") # tdSql.query("select distinct jtag->'tag1' from jsons1") # tdSql.checkRows(8) # tdSql.query("select distinct jtag from jsons1") # tdSql.checkRows(9) # # #test dumplicate key with normal colomn - # tdSql.execute("INSERT INTO jsons1_15 using jsons1 tags('{\"tbname\":\"tt\",\"databool\":true,\"datastr\":\"是是是\"}') values(1591060828000, 4, false, 'jjsf', \"你就会\")") + tdSql.execute("INSERT INTO jsons1_15 using jsons1 tags('{\"tbname\":\"tt\",\"databool\":true,\"datastr\":\"是是是\"}') values(1591060828000, 4, false, 'jjsf', \"你就会\")") # tdSql.query("select *,tbname,jtag from jsons1 where jtag->'datastr' match '是' and datastr match 'js'") # tdSql.checkRows(1) # tdSql.query("select tbname,jtag->'tbname' from jsons1 where jtag->'tbname'='tt' and tbname='jsons1_14'") From b1a781cc09a6eac763c9689249c129fa60d13d4e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 4 Jun 2022 12:42:25 +0000 Subject: [PATCH 17/63] more progress --- include/common/tdataformat.h | 7 +++ source/dnode/vnode/src/inc/tsdb.h | 26 ++++++++++ source/dnode/vnode/src/tsdb/tsdbCommit2.c | 55 +++++++++++++++++++++ source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 22 --------- 4 files changed, 88 insertions(+), 22 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 6a73d3d678..2e69640a06 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -34,6 +34,7 @@ typedef struct SValue SValue; typedef struct SColVal SColVal; typedef struct STSRow2 STSRow2; typedef struct STSRowBuilder STSRowBuilder; +typedef struct SColData SColData; typedef struct STagVal STagVal; typedef struct STag STag; @@ -169,6 +170,12 @@ struct STag { }; #pragma pack(pop) +struct SColData { + int16_t cid; + uint32_t nData; + uint8_t *pData; +}; + #if 1 //================================================================================================================================================ // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 3a2eec30ba..cb12550d1f 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -34,6 +34,7 @@ extern "C" { typedef struct TSDBROW TSDBROW; typedef struct TSDBKEY TSDBKEY; +typedef struct TABLEID TABLEID; typedef struct SDelOp SDelOp; static int tsdbKeyCmprFn(const void *p1, const void *p2); @@ -883,6 +884,11 @@ struct TSDBKEY { TSKEY ts; }; +struct TABLEID { + tb_uid_t suid; + tb_uid_t uid; +}; + struct SDelOp { int64_t version; TSKEY sKey; // included @@ -919,6 +925,26 @@ static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) { return 0; } +typedef struct SMemSkipListNode SMemSkipListNode; +typedef struct SMemSkipList { + uint32_t seed; + int32_t size; + int8_t maxLevel; + int8_t level; + SMemSkipListNode *pHead; + SMemSkipListNode *pTail; +} SMemSkipList; + +struct SMemData { + tb_uid_t suid; + tb_uid_t uid; + TSDBKEY minKey; + TSDBKEY maxKey; + SDelOp *delOpHead; + SDelOp *delOpTail; + SMemSkipList sl; +}; + struct SMemDataIter { SMemData *pMemData; TSDBROW row; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 6bb53a1263..5741da0e80 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -88,6 +88,30 @@ static int32_t tsdbEndCommit(SCommitH *pCHandle) { return code; } +static int32_t tsdbCommitTableData(SCommitH *pCHandle, SMemData *pMemData, SBlockIdx *pBlockIdx) { + int32_t code = 0; + // TODO + return code; +} + +static int32_t tsdbTableIdCmprFn(const void *p1, const void *p2) { + TABLEID *pId1 = (TABLEID *)p1; + TABLEID *pId2 = (TABLEID *)p2; + + if (pId1->suid < pId2->suid) { + return -1; + } else if (pId1->suid > pId2->suid) { + return 1; + } + + if (pId1->uid < pId2->uid) { + return -1; + } else if (pId1->uid > pId2->uid) { + return 1; + } + + return 0; +} static int32_t tsdbCommitToFile(SCommitH *pCHandle, int32_t fid) { int32_t code = 0; SMemDataIter iter = {0}; @@ -114,6 +138,8 @@ static int32_t tsdbCommitToFile(SCommitH *pCHandle, int32_t fid) { if (!hasData) return code; + // create or open the file to commit(todo) + // loop to commit each table data nBlockIdx = 0; for (;;) { @@ -127,7 +153,36 @@ static int32_t tsdbCommitToFile(SCommitH *pCHandle, int32_t fid) { if (iBlockIdx < nBlockIdx) { // pBlockIdx } + + if (pMemData && pBlockIdx) { + int32_t c = tsdbTableIdCmprFn(&(TABLEID){.suid = pMemData->suid, .uid = pMemData->uid}, + &(TABLEID){.suid = pBlockIdx->suid, .uid = pBlockIdx->uid}); + if (c == 0) { + iMemData++; + iBlockIdx++; + } else if (c < 0) { + pBlockIdx = NULL; + iMemData++; + } else { + pMemData = NULL; + iBlockIdx++; + } + } else { + if (pMemData) { + iMemData++; + } else { + iBlockIdx++; + } + } + + code = tsdbCommitTableData(pCHandle, pMemData, pBlockIdx); + if (code) { + goto _err; + } } return code; + +_err: + return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 1c28b086b3..dee025cae6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,33 +15,11 @@ #include "tsdb.h" -typedef struct SMemSkipList SMemSkipList; -typedef struct SMemSkipListNode SMemSkipListNode; - struct SMemSkipListNode { int8_t level; SMemSkipListNode *forwards[0]; }; -struct SMemSkipList { - uint32_t seed; - int32_t size; - int8_t maxLevel; - int8_t level; - SMemSkipListNode *pHead; - SMemSkipListNode *pTail; -}; - -struct SMemData { - tb_uid_t suid; - tb_uid_t uid; - TSDBKEY minKey; - TSDBKEY maxKey; - SDelOp *delOpHead; - SDelOp *delOpTail; - SMemSkipList sl; -}; - typedef struct { tb_uid_t uid; STSchema *pTSchema; From d261c17a4de8c9a787106ffb17dfe481f3ed5919 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 4 Jun 2022 21:31:07 +0800 Subject: [PATCH 18/63] fix: a problem of parser async --- source/libs/parser/inc/parInt.h | 1 - source/libs/parser/src/parAstParser.c | 15 ++-- source/libs/parser/src/parInsert.c | 64 +++++++++------- source/libs/parser/src/parTranslater.c | 91 ++++++++++++----------- source/libs/parser/src/parUtil.c | 21 ++++-- source/libs/parser/test/parInsertTest.cpp | 1 + 6 files changed, 110 insertions(+), 83 deletions(-) diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index 8ec20cde5a..3efe6700d2 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -32,7 +32,6 @@ int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery); int32_t translate(SParseContext* pParseCxt, SQuery* pQuery); int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); -int32_t isNotSchemalessDb(SParseContext* pContext, char *dbName); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 7abe244261..96fb810e5c 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -175,11 +175,11 @@ static int32_t collectMetaKeyFromSelect(SCollectMetaKeyCxt* pCxt, SSelectStmt* p } static int32_t collectMetaKeyFromCreateTable(SCollectMetaKeyCxt* pCxt, SCreateTableStmt* pStmt) { - if (NULL == pStmt->pTags) { - return reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); - } else { - return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); + int32_t code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); + if (TSDB_CODE_SUCCESS == code && NULL == pStmt->pTags) { + code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); } + return code; } static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCreateMultiTableStmt* pStmt) { @@ -187,8 +187,11 @@ static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCre SNode* pNode = NULL; FOREACH(pNode, pStmt->pSubTables) { SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode; - code = - reserveTableMetaInCache(pCxt->pParseCxt->acctId, pClause->useDbName, pClause->useTableName, pCxt->pMetaCache); + code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pClause->dbName, pCxt->pMetaCache); + if (TSDB_CODE_SUCCESS == code) { + code = + reserveTableMetaInCache(pCxt->pParseCxt->acctId, pClause->useDbName, pClause->useTableName, pCxt->pMetaCache); + } if (TSDB_CODE_SUCCESS == code) { code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, pCxt->pMetaCache); } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 422c480397..efe0bd75b2 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -254,7 +254,7 @@ static int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, con static int32_t checkAuth(SInsertParseContext* pCxt, char* pDbFname, bool* pPass) { SParseContext* pBasicCtx = pCxt->pComCxt; - if (NULL != pCxt->pMetaCache) { + if (pBasicCtx->async) { return getUserAuthFromCache(pCxt->pMetaCache, pBasicCtx->pUser, pDbFname, AUTH_TYPE_WRITE, pPass); } return catalogChkAuth(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, pBasicCtx->pUser, pDbFname, @@ -263,7 +263,7 @@ static int32_t checkAuth(SInsertParseContext* pCxt, char* pDbFname, bool* pPass) static int32_t getTableSchema(SInsertParseContext* pCxt, SName* pTbName, bool isStb, STableMeta** pTableMeta) { SParseContext* pBasicCtx = pCxt->pComCxt; - if (NULL != pCxt->pMetaCache) { + if (pBasicCtx->async) { return getTableMetaFromCache(pCxt->pMetaCache, pTbName, pTableMeta); } if (isStb) { @@ -275,7 +275,7 @@ static int32_t getTableSchema(SInsertParseContext* pCxt, SName* pTbName, bool is static int32_t getTableVgroup(SInsertParseContext* pCxt, SName* pTbName, SVgroupInfo* pVg) { SParseContext* pBasicCtx = pCxt->pComCxt; - if (NULL != pCxt->pMetaCache) { + if (pBasicCtx->async) { return getTableVgroupFromCache(pCxt->pMetaCache, pTbName, pVg); } return catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, pTbName, pVg); @@ -305,6 +305,16 @@ static int32_t getSTableMeta(SInsertParseContext* pCxt, SName* name, char* dbFna return getTableMetaImpl(pCxt, name, dbFname, true); } +static int32_t getDBCfg(SInsertParseContext* pCxt, const char* pDbFName, SDbCfgInfo* pInfo) { + SParseContext* pBasicCtx = pCxt->pComCxt; + if (pBasicCtx->async) { + CHECK_CODE(getDbCfgFromCache(pCxt->pMetaCache, pDbFName, pInfo)); + } else { + CHECK_CODE(catalogGetDBCfg(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, pDbFName, pInfo)); + } + return TSDB_CODE_SUCCESS; +} + static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchema) { while (start < end) { if (strlen(pSchema[start].name) == pColname->n && strncmp(pColname->z, pSchema[start].name, pColname->n) == 0) { @@ -1090,10 +1100,10 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SName* name, char* tb SName sname; createSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); - char stbFName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(&sname, stbFName); + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(&sname, dbFName); - CHECK_CODE(getSTableMeta(pCxt, &sname, stbFName)); + CHECK_CODE(getSTableMeta(pCxt, &sname, dbFName)); if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) { return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); } @@ -1282,6 +1292,14 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) { destroyBlockArrayList(pCxt->pVgDataBlocks); } +static int32_t checkSchemalessDb(SInsertParseContext* pCxt, char* pDbName) { + SDbCfgInfo pInfo = {0}; + char fullName[TSDB_TABLE_FNAME_LEN]; + snprintf(fullName, sizeof(fullName), "%d.%s", pCxt->pComCxt->acctId, pDbName); + CHECK_CODE(getDBCfg(pCxt, fullName, &pInfo)); + return pInfo.schemaless ? TSDB_CODE_SML_INVALID_DB_CONF : TSDB_CODE_SUCCESS; +} + // tb_name // [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] // [(field1_name, ...)] @@ -1335,7 +1353,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { SName name; CHECK_CODE(createSName(&name, &tbnameToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); - CHECK_CODE(isNotSchemalessDb(pCxt->pComCxt, name.dbname)); + CHECK_CODE(checkSchemalessDb(pCxt, name.dbname)); tNameExtractFullName(&name, tbFName); CHECK_CODE(taosHashPut(pCxt->pTableNameHashObj, tbFName, strlen(tbFName), &name, sizeof(SName))); @@ -1413,23 +1431,6 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildOutput(pCxt); } -int32_t isNotSchemalessDb(SParseContext* pContext, char *dbName){ - SName name; - tNameSetDbName(&name, pContext->acctId, dbName, strlen(dbName)); - char dbFname[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(&name, dbFname); - SDbCfgInfo pInfo = {0}; - int32_t code = catalogGetDBCfg(pContext->pCatalog, pContext->pTransporter, &pContext->mgmtEpSet, dbFname, &pInfo); - if (code != TSDB_CODE_SUCCESS) { - parserError("catalogGetDBCfg error, code:%s, dbFName:%s", tstrerror(code), dbFname); - return code; - } - if (pInfo.schemaless){ - parserError("can not insert into schemaless db:%s", dbFname); - return TSDB_CODE_SML_INVALID_DB_CONF; - } - return TSDB_CODE_SUCCESS; -} // INSERT INTO // tb_name // [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] @@ -1472,6 +1473,8 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { if (NULL == *pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } + } else { + context.pMetaCache = (*pQuery)->pMetaCache; } (*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE; (*pQuery)->haveResultSet = false; @@ -1586,12 +1589,20 @@ static int32_t skipUsingClause(SInsertParseSyntaxCxt* pCxt) { static int32_t collectTableMetaKey(SInsertParseSyntaxCxt* pCxt, SToken* pTbToken) { SName name; CHECK_CODE(createSName(&name, pTbToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); + CHECK_CODE(reserveDbCfgInCache(pCxt->pComCxt->acctId, name.dbname, pCxt->pMetaCache)); CHECK_CODE(reserveUserAuthInCacheExt(pCxt->pComCxt->pUser, &name, AUTH_TYPE_WRITE, pCxt->pMetaCache)); CHECK_CODE(reserveTableMetaInCacheExt(&name, pCxt->pMetaCache)); CHECK_CODE(reserveTableVgroupInCacheExt(&name, pCxt->pMetaCache)); return TSDB_CODE_SUCCESS; } +static int32_t collectAutoCreateTableMetaKey(SInsertParseSyntaxCxt* pCxt, SToken* pTbToken) { + SName name; + CHECK_CODE(createSName(&name, pTbToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); + CHECK_CODE(reserveTableVgroupInCacheExt(&name, pCxt->pMetaCache)); + return TSDB_CODE_SUCCESS; +} + static int32_t parseInsertBodySyntax(SInsertParseSyntaxCxt* pCxt) { bool hasData = false; // for each table @@ -1620,6 +1631,7 @@ static int32_t parseInsertBodySyntax(SInsertParseSyntaxCxt* pCxt) { // USING clause if (TK_USING == sToken.type) { + CHECK_CODE(collectAutoCreateTableMetaKey(pCxt, &tbnameToken)); NEXT_TOKEN(pCxt->pSql, sToken); CHECK_CODE(collectTableMetaKey(pCxt, &sToken)); CHECK_CODE(skipUsingClause(pCxt)); @@ -2172,8 +2184,8 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p val.nData = kv->length; } else if (pTagSchema->type == TSDB_DATA_TYPE_NCHAR) { int32_t output = 0; - void *p = taosMemoryCalloc(1, kv->length * TSDB_NCHAR_SIZE); - if(p == NULL){ + void* p = taosMemoryCalloc(1, kv->length * TSDB_NCHAR_SIZE); + if (p == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto end; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 2d37e9087d..69e45e2dcf 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -829,8 +829,9 @@ static EDealRes translateComparisonOperator(STranslateContext* pCxt, SOperatorNo if (!IS_VAR_DATA_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); } - if (QUERY_NODE_VALUE != nodeType(pOp->pRight) || - ((!IS_STR_DATA_TYPE(((SExprNode*)(pOp->pRight))->resType.type)) && (((SExprNode*)(pOp->pRight))->resType.type != TSDB_DATA_TYPE_NULL))) { + if (QUERY_NODE_VALUE != nodeType(pOp->pRight) || + ((!IS_STR_DATA_TYPE(((SExprNode*)(pOp->pRight))->resType.type)) && + (((SExprNode*)(pOp->pRight))->resType.type != TSDB_DATA_TYPE_NULL))) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } } @@ -2632,8 +2633,23 @@ static int32_t checkTableSchema(STranslateContext* pCxt, SCreateTableStmt* pStmt return code; } +static int32_t checkSchemalessDb(STranslateContext* pCxt, const char* pDbName) { + if (0 != pCxt->pParseCxt->schemalessType) { + return TSDB_CODE_SUCCESS; + } + SDbCfgInfo info = {0}; + int32_t code = getDBCfg(pCxt, pDbName, &info); + if (TSDB_CODE_SUCCESS == code) { + code = info.schemaless ? TSDB_CODE_SML_INVALID_DB_CONF : TSDB_CODE_SUCCESS; + } + return code; +} + static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) { - int32_t code = checTableFactorOption(pCxt, pStmt->pOptions->filesFactor); + int32_t code = checkSchemalessDb(pCxt, pStmt->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = checTableFactorOption(pCxt, pStmt->pOptions->filesFactor); + } if (TSDB_CODE_SUCCESS == code) { code = checkTableRollupOption(pCxt, pStmt->pOptions->pRollupFuncs); } @@ -2646,11 +2662,6 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt if (TSDB_CODE_SUCCESS == code) { code = checkTableSchema(pCxt, pStmt); } - if (TSDB_CODE_SUCCESS == code) { - if(pCxt->pParseCxt->schemalessType == 0){ - code = isNotSchemalessDb(pCxt->pParseCxt, pStmt->dbName); - } - } return code; } @@ -4238,7 +4249,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla int16_t nTags = 0, nBufPos = 0; SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta); SNode * pTag = NULL, *pNode = NULL; - bool isJson = false; + bool isJson = false; FORBOTH(pTag, pStmt->pSpecificTags, pNode, pStmt->pValsOfTags) { SColumnNode* pCol = (SColumnNode*)pTag; SSchema* pSchema = NULL; @@ -4271,11 +4282,11 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla isJson = true; code = parseJsontoTagData(pVal->literal, pTagArray, ppTag, &pCxt->msgBuf); - if(code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { goto end; } - }else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { - void* nodeVal = nodesGetValueFromNode(pVal); + } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { + void* nodeVal = nodesGetValueFromNode(pVal); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; if (IS_VAR_DATA_TYPE(pTagSchema->type)) { val.pData = varDataVal(nodeVal); @@ -4287,13 +4298,13 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla } } - if(!isJson) code = tTagNew(pTagArray, 1, false, ppTag); + if (!isJson) code = tTagNew(pTagArray, 1, false, ppTag); end: - if(isJson){ + if (isJson) { for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { - STagVal *p = (STagVal *)taosArrayGet(pTagArray, i); - if(IS_VAR_DATA_TYPE(p->type)){ + STagVal* p = (STagVal*)taosArrayGet(pTagArray, i); + if (IS_VAR_DATA_TYPE(p->type)) { taosMemoryFree(p->pData); } } @@ -4338,11 +4349,11 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau isJson = true; code = parseJsontoTagData(pVal->literal, pTagArray, ppTag, &pCxt->msgBuf); - if(code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { goto end; } - }else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { - char* tmpVal = nodesGetValueFromNode(pVal); + } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { + char* tmpVal = nodesGetValueFromNode(pVal); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; if (IS_VAR_DATA_TYPE(pTagSchema->type)) { val.pData = varDataVal(tmpVal); @@ -4354,13 +4365,13 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau } ++index; } - if(!isJson) code = tTagNew(pTagArray, 1, false, ppTag); + if (!isJson) code = tTagNew(pTagArray, 1, false, ppTag); end: - if(isJson){ + if (isJson) { for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { - STagVal *p = (STagVal *)taosArrayGet(pTagArray, i); - if(IS_VAR_DATA_TYPE(p->type)){ + STagVal* p = (STagVal*)taosArrayGet(pTagArray, i); + if (IS_VAR_DATA_TYPE(p->type)) { taosMemoryFree(p->pData); } } @@ -4428,7 +4439,6 @@ static SArray* serializeVgroupsCreateTableBatch(int32_t acctId, SHashObj* pVgrou } static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) { - SCreateMultiTableStmt* pStmt = (SCreateMultiTableStmt*)pQuery->pRoot; SHashObj* pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -4439,11 +4449,11 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) int32_t code = TSDB_CODE_SUCCESS; SNode* pNode; FOREACH(pNode, pStmt->pSubTables) { - if(pCxt->pParseCxt->schemalessType == 0 && - (code = isNotSchemalessDb(pCxt->pParseCxt, ((SCreateSubTableClause*)pNode)->dbName)) != TSDB_CODE_SUCCESS){ - return code; + SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode; + code = checkSchemalessDb(pCxt, pClause->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = rewriteCreateSubTable(pCxt, pClause, pVgroupHashmap); } - code = rewriteCreateSubTable(pCxt, (SCreateSubTableClause*)pNode, pVgroupHashmap); if (TSDB_CODE_SUCCESS != code) { taosHashCleanup(pVgroupHashmap); return code; @@ -4633,27 +4643,27 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS strlen(pStmt->pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pStmt->pVal->literal); } - SArray *pTagVals = taosArrayInit(1, sizeof(STagVal)); + SArray* pTagVals = taosArrayInit(1, sizeof(STagVal)); int32_t code = TSDB_CODE_SUCCESS; - STag* pTag = NULL; - do{ + STag* pTag = NULL; + do { code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pTag, &pCxt->msgBuf); if (TSDB_CODE_SUCCESS != code) { break; } - }while(0); + } while (0); for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { - STagVal *p = (STagVal *)taosArrayGet(pTagVals, i); - if(IS_VAR_DATA_TYPE(p->type)){ + STagVal* p = (STagVal*)taosArrayGet(pTagVals, i); + if (IS_VAR_DATA_TYPE(p->type)) { taosMemoryFree(p->pData); } } taosArrayDestroy(pTagVals); - if (code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { return code; } pReq->nTagVal = pTag->len; - pReq->pTagVal = (uint8_t *)pTag; + pReq->pTagVal = (uint8_t*)pTag; pStmt->pVal->datum.p = (char*)pTag; // for free } else { pReq->nTagVal = pStmt->pVal->node.resType.bytes; @@ -4854,13 +4864,8 @@ static int32_t buildModifyVnodeArray(STranslateContext* pCxt, SAlterTableStmt* p static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) { SAlterTableStmt* pStmt = (SAlterTableStmt*)pQuery->pRoot; - int32_t code = TSDB_CODE_SUCCESS; - if(pCxt->pParseCxt->schemalessType == 0 && - (code = isNotSchemalessDb(pCxt->pParseCxt, pStmt->dbName)) != TSDB_CODE_SUCCESS){ - return code; - } - - STableMeta* pTableMeta = NULL; + int32_t code = checkSchemalessDb(pCxt, pStmt->dbName); + STableMeta* pTableMeta = NULL; code = getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pTableMeta); if (TSDB_CODE_SUCCESS != code) { return code; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 0a1915d6c2..4896b64686 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -322,11 +322,11 @@ static bool isValidateTag(char* input) { return true; } -int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMsgBuf* pMsgBuf) { +int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, SMsgBuf* pMsgBuf) { int32_t retCode = TSDB_CODE_SUCCESS; - cJSON* root = NULL; + cJSON* root = NULL; SHashObj* keyHash = NULL; - int32_t size = 0; + int32_t size = 0; // set json NULL data if (!json || strtrim((char*)json) == 0 || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0) { retCode = TSDB_CODE_SUCCESS; @@ -371,7 +371,8 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMs } STagVal val = {0}; val.pKey = jsonKey; - taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless + taosHashPut(keyHash, jsonKey, keyLen, &keyLen, + CHAR_BYTES); // add key to hash to remove dumplicate, value is useless if (item->type == cJSON_String) { // add json value format: type|data char* jsonValue = item->valuestring; @@ -382,8 +383,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMs goto end; } val.type = TSDB_DATA_TYPE_NCHAR; - if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)tmp, - (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) { + if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)tmp, (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) { uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno)); retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue); @@ -413,7 +413,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMs end: taosHashCleanup(keyHash); - if(retCode == TSDB_CODE_SUCCESS){ + if (retCode == TSDB_CODE_SUCCESS) { tTagNew(pTagVals, 1, true, ppTag); } cJSON_Delete(root); @@ -679,6 +679,7 @@ int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, S tNameExtractFullName(pName, fullName); STableMeta** pRes = taosHashGet(pMetaCache->pTableMeta, fullName, strlen(fullName)); if (NULL == pRes || NULL == *pRes) { + parserError("getTableMetaFromCache error: %s", fullName); return TSDB_CODE_PAR_INTERNAL_ERROR; } *pMeta = tableMetaDup(*pRes); @@ -707,6 +708,7 @@ int32_t reserveDbVgInfoInCache(int32_t acctId, const char* pDb, SParseMetaCache* int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo) { SArray** pRes = taosHashGet(pMetaCache->pDbVgroup, pDbFName, strlen(pDbFName)); if (NULL == pRes) { + parserError("getDbVgInfoFromCache error: %s", pDbFName); return TSDB_CODE_PAR_INTERNAL_ERROR; } // *pRes is null, which is a legal value, indicating that the user DB has not been created @@ -734,6 +736,7 @@ int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, tNameExtractFullName(pName, fullName); SVgroupInfo** pRes = taosHashGet(pMetaCache->pTableVgroup, fullName, strlen(fullName)); if (NULL == pRes || NULL == *pRes) { + parserError("getTableVgroupFromCache error: %s", fullName); return TSDB_CODE_PAR_INTERNAL_ERROR; } memcpy(pVgroup, *pRes, sizeof(SVgroupInfo)); @@ -748,6 +751,7 @@ int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFNam int32_t* pTableNum) { SDbInfo** pRes = taosHashGet(pMetaCache->pDbCfg, pDbFName, strlen(pDbFName)); if (NULL == pRes || NULL == *pRes) { + parserError("getDbVgVersionFromCache error: %s", pDbFName); return TSDB_CODE_PAR_INTERNAL_ERROR; } *pVersion = (*pRes)->vgVer; @@ -763,6 +767,7 @@ int32_t reserveDbCfgInCache(int32_t acctId, const char* pDb, SParseMetaCache* pM int32_t getDbCfgFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SDbCfgInfo* pInfo) { SDbCfgInfo** pRes = taosHashGet(pMetaCache->pDbCfg, pDbFName, strlen(pDbFName)); if (NULL == pRes || NULL == *pRes) { + parserError("getDbCfgFromCache error: %s", pDbFName); return TSDB_CODE_PAR_INTERNAL_ERROR; } memcpy(pInfo, *pRes, sizeof(SDbCfgInfo)); @@ -801,6 +806,7 @@ int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, con int32_t len = userAuthToStringExt(pUser, pDbFName, type, key); bool* pRes = taosHashGet(pMetaCache->pUserAuth, key, len); if (NULL == pRes) { + parserError("getUserAuthFromCache error: %s, %s, %d", pUser, pDbFName, type); return TSDB_CODE_PAR_INTERNAL_ERROR; } *pPass = *pRes; @@ -820,6 +826,7 @@ int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache) { int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFuncInfo* pInfo) { SFuncInfo** pRes = taosHashGet(pMetaCache->pUdf, pFunc, strlen(pFunc)); if (NULL == pRes || NULL == *pRes) { + parserError("getUdfInfoFromCache error: %s", pFunc); return TSDB_CODE_PAR_INTERNAL_ERROR; } memcpy(pInfo, *pRes, sizeof(SFuncInfo)); diff --git a/source/libs/parser/test/parInsertTest.cpp b/source/libs/parser/test/parInsertTest.cpp index 4d313fca76..eae83e2223 100644 --- a/source/libs/parser/test/parInsertTest.cpp +++ b/source/libs/parser/test/parInsertTest.cpp @@ -59,6 +59,7 @@ class InsertTest : public Test { } int32_t runAsync() { + cxt_.async = true; code_ = parseInsertSyntax(&cxt_, &res_); if (code_ != TSDB_CODE_SUCCESS) { cout << "parseInsertSyntax code:" << toString(code_) << ", msg:" << errMagBuf_ << endl; From af4aaed01f9ba67d6fbbd86aaae547bb5e3733c7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 4 Jun 2022 14:03:05 +0000 Subject: [PATCH 19/63] more --- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 5741da0e80..abeca274d4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -89,8 +89,17 @@ static int32_t tsdbEndCommit(SCommitH *pCHandle) { } static int32_t tsdbCommitTableData(SCommitH *pCHandle, SMemData *pMemData, SBlockIdx *pBlockIdx) { - int32_t code = 0; - // TODO + int32_t code = 0; + SMemDataIter iter = {0}; + + if (pMemData && pBlockIdx) { + // merge + } else if (pMemData) { + // new one + } else { + // save old ones + } + return code; } From 17ea0e489afbc0094be0a980ceb3f920755b4bc1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 4 Jun 2022 22:05:14 +0800 Subject: [PATCH 20/63] feat: add sort/group logic for json --- tests/system-test/2-query/json_tag.py | 178 +++++++++++++------------- 1 file changed, 91 insertions(+), 87 deletions(-) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index f6a32cbe95..8f67dd7d42 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -345,10 +345,10 @@ class TDTestCase: # # # test distinct tdSql.execute("insert into jsons1_14 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":null}') values(1591062628000, 2, NULL, '你就会', 'dws')") - # tdSql.query("select distinct jtag->'tag1' from jsons1") - # tdSql.checkRows(8) - # tdSql.query("select distinct jtag from jsons1") - # tdSql.checkRows(9) + tdSql.query("select distinct jtag->'tag1' from jsons1") + tdSql.checkRows(8) + tdSql.query("select distinct jtag from jsons1") + tdSql.checkRows(9) # # #test dumplicate key with normal colomn tdSql.execute("INSERT INTO jsons1_15 using jsons1 tags('{\"tbname\":\"tt\",\"databool\":true,\"datastr\":\"是是是\"}') values(1591060828000, 4, false, 'jjsf', \"你就会\")") @@ -376,67 +376,71 @@ class TDTestCase: # cname_list.append("a.jtag->'tag3'") # tdSql.checkColNameList(res, cname_list) # - # # test group by & order by json tag - # tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag->'tag2'") - # tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag") - # tdSql.query("select count(*) from jsons1 group by jtag->'tag1' order by jtag->'tag1' desc") - # tdSql.checkRows(8) - # tdSql.checkData(1, 0, 2) - # tdSql.checkData(1, 1, '"femail"') - # tdSql.checkData(2, 0, 1) - # tdSql.checkData(2, 1, 11) - # tdSql.checkData(5, 0, 1) - # tdSql.checkData(5, 1, "false") + # test group by & order by json tag + tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag->'tag2'") + tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag") + tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' desc") + tdSql.checkRows(8) + tdSql.checkData(0, 0, 2) + tdSql.checkData(0, 1, '"femail"') + tdSql.checkData(1, 0, 2) + tdSql.checkData(1, 1, '"收到货"') + tdSql.checkData(2, 0, 1) + tdSql.checkData(2, 1, "11.000000000") + tdSql.checkData(5, 0, 1) + tdSql.checkData(5, 1, "false") # tdSql.checkData(6, 0, 1) # tdSql.checkData(6, 1, "null") - # tdSql.checkData(7, 0, 2) - # tdSql.checkData(7, 1, None) + tdSql.checkData(7, 0, 2) + tdSql.checkData(7, 1, None) + + tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' asc") + tdSql.checkRows(8) + tdSql.checkData(0, 0, 2) + tdSql.checkData(0, 1, None) + tdSql.checkData(2, 0, 1) + tdSql.checkData(2, 1, "false") + tdSql.checkData(5, 0, 1) + tdSql.checkData(5, 1, "11.000000000") + tdSql.checkData(7, 0, 2) + tdSql.checkData(7, 1, '"femail"') # - # tdSql.query("select count(*) from jsons1 group by jtag->'tag1' order by jtag->'tag1' asc") - # tdSql.checkRows(8) - # tdSql.checkData(0, 0, 2) + # test stddev with group by json tag + tdSql.query("select stddev(dataint),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") + tdSql.checkRows(8) + tdSql.checkData(0, 0, 10) # tdSql.checkData(0, 1, None) - # tdSql.checkData(2, 0, 1) - # tdSql.checkData(2, 1, "false") - # tdSql.checkData(5, 0, 1) - # tdSql.checkData(5, 1, 11) - # tdSql.checkData(6, 0, 2) - # tdSql.checkData(6, 1, '"femail"') - # - # # test stddev with group by json tag - # tdSql.query("select stddev(dataint) from jsons1 group by jtag->'tag1'") - # tdSql.checkData(0, 0, 10) - # tdSql.checkData(0, 1, None) - # tdSql.checkData(1, 0, 0) - # tdSql.checkData(1, 1, "null") - # tdSql.checkData(6, 0, 11) - # tdSql.checkData(6, 1, '"femail"') - # - # res = tdSql.getColNameList("select stddev(dataint) from jsons1 group by jsons1.jtag->'tag1'") - # cname_list = [] - # cname_list.append("stddev(dataint)") - # cname_list.append("jsons1.jtag->'tag1'") - # tdSql.checkColNameList(res, cname_list) - # - # # test top/bottom with group by json tag - # tdSql.query("select top(dataint,100) from jsons1 group by jtag->'tag1'") - # tdSql.checkRows(11) - # tdSql.checkData(0, 1, 4) - # tdSql.checkData(1, 1, 24) - # tdSql.checkData(1, 2, None) - # tdSql.checkData(8, 1, 1) - # tdSql.checkData(8, 2, '"femail"') - # - # # test having + tdSql.checkData(4, 0, 0) + tdSql.checkData(4, 1, "5.000000000") + tdSql.checkData(7, 0, 11) + tdSql.checkData(7, 1, '"femail"') + + res = tdSql.getColNameList("select stddev(dataint),jsons1.jtag->'tag1' from jsons1 group by jsons1.jtag->'tag1' order by jtag->'tag1'") + cname_list = [] + cname_list.append("stddev(dataint)") + cname_list.append("jsons1.jtag->'tag1'") + tdSql.checkColNameList(res, cname_list) + + # test top/bottom with group by json tag + tdSql.query("select top(dataint,2),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") + tdSql.checkRows(11) + tdSql.checkData(3, 0, 3) + tdSql.checkData(3, 1, "false") + # tdSql.checkData(3, 0, 24) + # tdSql.checkData(3, 1, None) + tdSql.checkData(10, 0, 23) + tdSql.checkData(10, 1, '"femail"') + + # test having # tdSql.query("select stddev(dataint) from jsons1 group by jtag->'tag1' having stddev(dataint) > 0") # tdSql.checkRows(2) - # - # # subquery with json tag - # tdSql.query("select * from (select jtag, dataint from jsons1)") - # tdSql.checkRows(11) - # tdSql.checkData(1, 1, 1) - # tdSql.checkData(2, 0, '{"tag1":5,"tag2":"beijing"}') - # + + # subquery with json tag + tdSql.query("select * from (select jtag, dataint from jsons1)") + tdSql.checkRows(11) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 0, '{"tag1":5,"tag2":"beijing"}') + # tdSql.query("select jtag->'tag1' from (select jtag->'tag1', dataint from jsons1)") # tdSql.checkRows(11) # tdSql.checkData(1, 0, '"femail"') @@ -513,17 +517,17 @@ class TDTestCase: # tdSql.checkRows(3) # # #test TD-12077 - # tdSql.execute("insert into jsons1_16 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":-2.111}') values(1591062628000, 2, NULL, '你就会', 'dws')") - # tdSql.query("select jtag->'tag3' from jsons1_16") - # tdSql.checkData(0, 0, '-2.111000000') - # + tdSql.execute("insert into jsons1_16 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":-2.111}') values(1591062628000, 2, NULL, '你就会', 'dws')") + tdSql.query("select jtag->'tag3' from jsons1_16") + tdSql.checkData(0, 0, '-2.111000000') + # # test TD-12452 - # tdSql.execute("ALTER TABLE jsons1_1 SET TAG jtag=NULL") - # tdSql.query("select jtag from jsons1_1") - # tdSql.checkData(0, 0, None) - # tdSql.execute("CREATE TABLE if not exists jsons1_20 using jsons1 tags(NULL)") - # tdSql.query("select jtag from jsons1_20") - # tdSql.checkData(0, 0, None) + tdSql.execute("ALTER TABLE jsons1_1 SET TAG jtag=NULL") + tdSql.query("select jtag from jsons1_1") + tdSql.checkData(0, 0, None) + tdSql.execute("CREATE TABLE if not exists jsons1_20 using jsons1 tags(NULL)") + tdSql.query("select jtag from jsons1_20") + tdSql.checkData(0, 0, None) # tdSql.execute("insert into jsons1_21 using jsons1 tags(NULL) values(1591061628000, 11, false, '你就会','')") # tdSql.query("select jtag from jsons1_21") # tdSql.checkData(0, 0, None) @@ -535,24 +539,24 @@ class TDTestCase: tdSql.checkData(5, 2, 4095) # # #test TD-13918 - # tdSql.execute("drop table if exists jsons_13918_1") - # tdSql.execute("drop table if exists jsons_13918_2") - # tdSql.execute("drop table if exists jsons_13918_3") - # tdSql.execute("drop table if exists jsons_13918_4") - # tdSql.execute("drop table if exists jsons_stb") - # tdSql.execute("create table jsons_stb (ts timestamp, dataInt int) tags (jtag json)") - # tdSql.error("create table jsons_13918_1 using jsons_stb tags ('nullx')") - # tdSql.error("create table jsons_13918_2 using jsons_stb tags (nullx)") - # tdSql.error("insert into jsons_13918_3 using jsons_stb tags('NULLx') values(1591061628001, 11)") - # tdSql.error("insert into jsons_13918_4 using jsons_stb tags(NULLx) values(1591061628002, 11)") - # tdSql.execute("create table jsons_13918_1 using jsons_stb tags ('null')") - # tdSql.execute("create table jsons_13918_2 using jsons_stb tags (null)") - # tdSql.execute("insert into jsons_13918_1 values(1591061628003, 11)") - # tdSql.execute("insert into jsons_13918_2 values(1591061628004, 11)") - # tdSql.execute("insert into jsons_13918_3 using jsons_stb tags('NULL') values(1591061628005, 11)") - # tdSql.execute("insert into jsons_13918_4 using jsons_stb tags(\"NULL\") values(1591061628006, 11)") - # tdSql.query("select * from jsons_stb") - # tdSql.checkRows(4) + tdSql.execute("drop table if exists jsons_13918_1") + tdSql.execute("drop table if exists jsons_13918_2") + tdSql.execute("drop table if exists jsons_13918_3") + tdSql.execute("drop table if exists jsons_13918_4") + tdSql.execute("drop table if exists jsons_stb") + tdSql.execute("create table jsons_stb (ts timestamp, dataInt int) tags (jtag json)") + tdSql.error("create table jsons_13918_1 using jsons_stb tags ('nullx')") + tdSql.error("create table jsons_13918_2 using jsons_stb tags (nullx)") + tdSql.error("insert into jsons_13918_3 using jsons_stb tags('NULLx') values(1591061628001, 11)") + tdSql.error("insert into jsons_13918_4 using jsons_stb tags(NULLx) values(1591061628002, 11)") + tdSql.execute("create table jsons_13918_1 using jsons_stb tags ('null')") + tdSql.execute("create table jsons_13918_2 using jsons_stb tags (null)") + tdSql.execute("insert into jsons_13918_1 values(1591061628003, 11)") + tdSql.execute("insert into jsons_13918_2 values(1591061628004, 11)") + tdSql.execute("insert into jsons_13918_3 using jsons_stb tags('NULL') values(1591061628005, 11)") + tdSql.execute("insert into jsons_13918_4 using jsons_stb tags(\"NULL\") values(1591061628006, 11)") + tdSql.query("select * from jsons_stb") + tdSql.checkRows(4) def stop(self): tdSql.close() From 32ee7629247f5d2b28ffe1c642971674ebb59b42 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 4 Jun 2022 14:27:34 +0000 Subject: [PATCH 21/63] add version on some structures --- source/dnode/vnode/src/inc/tsdb.h | 40 +++++------------- source/dnode/vnode/src/tsdb/tsdbCommit.c | 20 ++++----- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 1 + source/dnode/vnode/src/tsdb/tsdbRead.c | 47 +++++++++++----------- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 12 +++--- 5 files changed, 52 insertions(+), 68 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index cb12550d1f..acf83e4a9c 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -258,6 +258,11 @@ static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator *pIter) { // tsdbReadImpl typedef struct SReadH SReadH; +struct TSDBKEY { + int64_t version; + TSKEY ts; +}; + typedef struct { uint32_t len; uint32_t offset; @@ -265,25 +270,9 @@ typedef struct { uint32_t numOfBlocks : 30; uint64_t suid; uint64_t uid; - TSKEY maxKey; + TSDBKEY maxKey; } SBlockIdx; -#ifdef TD_REFACTOR_3 -typedef struct { - int64_t last : 1; - int64_t offset : 63; - int32_t algorithm : 8; - int32_t numOfRows : 24; - int32_t len; - int32_t keyLen; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols - int16_t numOfSubBlocks; - int16_t numOfCols; // not including timestamp column - TSKEY keyFirst; - TSKEY keyLast; -} SBlock; - -#else - typedef enum { TSDB_SBLK_VER_0 = 0, TSDB_SBLK_VER_MAX, @@ -306,17 +295,15 @@ typedef struct { int64_t offset; uint64_t aggrStat : 1; uint64_t aggrOffset : 63; - TSKEY keyFirst; - TSKEY keyLast; -} SBlockV0; - -#define SBlock SBlockV0 // latest SBlock definition + TSDBKEY minKey; + TSDBKEY maxKey; + // TSKEY keyFirst; + // TSKEY keyLast; +} SBlock; static FORCE_INLINE bool tsdbIsSupBlock(SBlock *pBlock) { return pBlock->numOfSubBlocks == 1; } static FORCE_INLINE bool tsdbIsSubBlock(SBlock *pBlock) { return pBlock->numOfSubBlocks == 0; } -#endif - typedef struct { int32_t delimiter; // For recovery usage int32_t tid; @@ -879,11 +866,6 @@ struct TSDBROW { STSRow2 tsRow; }; -struct TSDBKEY { - int64_t version; - TSKEY ts; -}; - struct TABLEID { tb_uid_t suid; tb_uid_t uid; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 928915a36d..3013f6deef 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -774,7 +774,7 @@ static int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, pIdx->uid = TABLE_UID(pTable); pIdx->hasLast = pBlock->last ? 1 : 0; - pIdx->maxKey = pBlock->keyLast; + pIdx->maxKey = pBlock->maxKey; pIdx->numOfBlocks = (uint32_t)nSupBlocks; pIdx->len = tlen; pIdx->offset = (uint32_t)offset; @@ -895,7 +895,7 @@ static int tsdbCommitToTable(SCommitH *pCommith, int tid) { return -1; } } else { - if (tsdbCommitMemData(pCommith, pIter, pBlock->keyFirst - 1, true) < 0) { + if (tsdbCommitMemData(pCommith, pIter, pBlock->minKey.ts - 1, true) < 0) { return -1; } } @@ -990,9 +990,9 @@ static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { TSKEY key = *(TSKEY *)arg1; SBlock *pBlock = (SBlock *)arg2; - if (key < pBlock->keyFirst) { + if (key < pBlock->minKey.ts) { return -1; - } else if (key > pBlock->keyLast) { + } else if (key > pBlock->maxKey.ts) { return 1; } else { return 0; @@ -1220,8 +1220,8 @@ static int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFi pBlock->numOfSubBlocks = isSuper ? 1 : 0; pBlock->numOfCols = nColsNotAllNull; pBlock->numOfBSma = nColsOfBlockSma; - pBlock->keyFirst = dataColsKeyFirst(pDataCols); - pBlock->keyLast = dataColsKeyLast(pDataCols); + pBlock->minKey.ts = dataColsKeyFirst(pDataCols); + pBlock->maxKey.ts = dataColsKeyLast(pDataCols); pBlock->aggrStat = aggrStatus; pBlock->blkVer = SBlockVerLatest; pBlock->aggrOffset = (uint64_t)offsetAggr; @@ -1229,7 +1229,7 @@ static int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFi tsdbDebug("vgId:%d, uid:%" PRId64 " a block of data is written to file %s, offset %" PRId64 " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, REPO_ID(pRepo), TABLE_UID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, - pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); + pBlock->numOfCols, pBlock->minKey.ts, pBlock->maxKey.ts); return 0; } @@ -1312,7 +1312,7 @@ static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) { if (bidx == nBlocks - 1) { keyLimit = pCommith->maxKey; } else { - keyLimit = pBlock[1].keyFirst - 1; + keyLimit = pBlock[1].minKey.ts - 1; } SSkipListIterator titer = *(pIter->pIter); @@ -1354,8 +1354,8 @@ static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) { } subBlocks[pBlock->numOfSubBlocks] = block; supBlock = *pBlock; - supBlock.keyFirst = mInfo.keyFirst; - supBlock.keyLast = mInfo.keyLast; + supBlock.minKey.ts = mInfo.keyFirst; + supBlock.maxKey.ts = mInfo.keyLast; supBlock.numOfSubBlocks++; supBlock.numOfRows = pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed; supBlock.offset = taosArrayGetSize(pCommith->aSubBlk) * sizeof(SBlock); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index abeca274d4..38e4795a83 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -17,6 +17,7 @@ typedef struct { SMemTable *pMemTable; + SReadH readh; SArray *aBlkIdx; } SCommitH; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5f2ea80078..95322f5eb3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -20,10 +20,10 @@ #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) -#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ - ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ - .numOfCols = (_block)->numOfCols, \ - .rows = (_block)->numOfRows, \ +#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ + ((SDataBlockInfo){.window = {.skey = (_block)->minKey.ts, .ekey = (_block)->maxKey.ts}, \ + .numOfCols = (_block)->numOfCols, \ + .rows = (_block)->numOfRows, \ .uid = (_checkInfo)->tableId}) enum { @@ -1105,12 +1105,12 @@ static int32_t binarySearchForBlock(SBlock* pBlock, int32_t numOfBlocks, TSKEY s if (numOfBlocks == 1) break; - if (skey > pBlock[midSlot].keyLast) { + if (skey > pBlock[midSlot].maxKey.ts) { if (numOfBlocks == 2) break; - if ((order == TSDB_ORDER_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break; + if ((order == TSDB_ORDER_DESC) && (skey < pBlock[midSlot + 1].minKey.ts)) break; firstSlot = midSlot + 1; - } else if (skey < pBlock[midSlot].keyFirst) { - if ((order == TSDB_ORDER_ASC) && (skey > pBlock[midSlot - 1].keyLast)) break; + } else if (skey < pBlock[midSlot].minKey.ts) { + if ((order == TSDB_ORDER_ASC) && (skey > pBlock[midSlot - 1].maxKey.ts)) break; lastSlot = midSlot - 1; } else { break; // got the slot @@ -1177,12 +1177,12 @@ static int32_t loadBlockInfo(STsdbReadHandle* pTsdbReadHandle, int32_t index, in int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); int32_t end = start; - if (s > pCompInfo->blocks[start].keyLast) { + if (s > pCompInfo->blocks[start].maxKey.ts) { return 0; } // todo speedup the procedure of located end block - while (end < (int32_t)compIndex->numOfBlocks && (pCompInfo->blocks[end].keyFirst <= e)) { + while (end < (int32_t)compIndex->numOfBlocks && (pCompInfo->blocks[end].minKey.ts <= e)) { end += 1; } @@ -1275,7 +1275,7 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl pBlock->numOfRows = pCols->numOfRows; // Convert from TKEY to TSKEY for primary timestamp column if current block has timestamp before 1970-01-01T00:00:00Z - if (pBlock->keyFirst < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { + if (pBlock->minKey.ts < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { int64_t* src = pCols->cols[0].pData; for (int32_t i = 0; i < pBlock->numOfRows; ++i) { src[i] = tdGetKey(src[i]); @@ -1287,7 +1287,7 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl tsdbDebug("%p load file block into buffer, index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%" PRId64 " us, %s", - pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, + pTsdbReadHandle, slotIndex, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->numOfRows, elapsedTime, pTsdbReadHandle->idStr); return TSDB_CODE_SUCCESS; @@ -1295,7 +1295,8 @@ _error: pBlock->numOfRows = 0; tsdbError("%p error occurs in loading file block, index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, %s", - pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pTsdbReadHandle->idStr); + pTsdbReadHandle, slotIndex, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->numOfRows, + pTsdbReadHandle->idStr); return terrno; } @@ -1423,7 +1424,7 @@ static int32_t loadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBloc if (asc) { // query ended in/started from current block - if (pTsdbReadHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { + if (pTsdbReadHandle->window.ekey < pBlock->maxKey.ts || pCheckInfo->lastKey > pBlock->minKey.ts) { if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { *exists = false; return code; @@ -1432,35 +1433,35 @@ static int32_t loadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBloc SDataCols* pTSCol = pTsdbReadHandle->rhelper.pDCols[0]; assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows); - if (pCheckInfo->lastKey > pBlock->keyFirst) { + if (pCheckInfo->lastKey > pBlock->minKey.ts) { cur->pos = binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pTsdbReadHandle->order); } else { cur->pos = 0; } - assert(pCheckInfo->lastKey <= pBlock->keyLast); + assert(pCheckInfo->lastKey <= pBlock->maxKey.ts); doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock); } else { // the whole block is loaded in to buffer cur->pos = asc ? 0 : (pBlock->numOfRows - 1); code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlock, pCheckInfo); } } else { // desc order, query ended in current block - if (pTsdbReadHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) { + if (pTsdbReadHandle->window.ekey > pBlock->minKey.ts || pCheckInfo->lastKey < pBlock->maxKey.ts) { if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { *exists = false; return code; } SDataCols* pTsCol = pTsdbReadHandle->rhelper.pDCols[0]; - if (pCheckInfo->lastKey < pBlock->keyLast) { + if (pCheckInfo->lastKey < pBlock->maxKey.ts) { cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pTsdbReadHandle->order); } else { cur->pos = pBlock->numOfRows - 1; } - assert(pCheckInfo->lastKey >= pBlock->keyFirst); + assert(pCheckInfo->lastKey >= pBlock->minKey.ts); doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock); } else { cur->pos = asc ? 0 : (pBlock->numOfRows - 1); @@ -1981,8 +1982,8 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf cur->pos >= 0 && cur->pos < pBlock->numOfRows); // Even Multi-Version supported, the records with duplicated TSKEY would be merged inside of tsdbLoadData interface. TSKEY* tsArray = pCols->cols[0].pData; - assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst && - tsArray[pBlock->numOfRows - 1] == pBlock->keyLast); + assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->minKey.ts && + tsArray[pBlock->numOfRows - 1] == pBlock->maxKey.ts); bool ascScan = ASCENDING_TRAVERSE(pTsdbReadHandle->order); int32_t step = ascScan ? 1 : -1; @@ -3576,8 +3577,8 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SColumnDat assert(pPrimaryColStatis->colId == PRIMARYKEY_TIMESTAMP_COL_ID); pPrimaryColStatis->numOfNull = 0; - pPrimaryColStatis->min = pBlockInfo->compBlock->keyFirst; - pPrimaryColStatis->max = pBlockInfo->compBlock->keyLast; + pPrimaryColStatis->min = pBlockInfo->compBlock->minKey.ts; + pPrimaryColStatis->max = pBlockInfo->compBlock->maxKey.ts; pHandle->suppInfo.plist[0] = &pHandle->suppInfo.pstatis[0]; // update the number of NULL data rows diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index a6f2ff1394..e333f0566b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -339,8 +339,8 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { } ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows); - ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst); - ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast); + ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->minKey.ts); + ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->maxKey.ts); return 0; } @@ -457,8 +457,8 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, } ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows); - ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst); - ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast); + ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->minKey.ts); + ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->maxKey.ts); return 0; } @@ -559,7 +559,7 @@ int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx) { tlen += taosEncodeFixedU8(buf, pIdx->hasLast); tlen += taosEncodeVariantU32(buf, pIdx->numOfBlocks); tlen += taosEncodeFixedU64(buf, pIdx->uid); - tlen += taosEncodeFixedU64(buf, pIdx->maxKey); + tlen += taosEncodeFixedU64(buf, pIdx->maxKey.ts); return tlen; } @@ -579,7 +579,7 @@ void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx) { if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; pIdx->uid = (int64_t)value; if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; - pIdx->maxKey = (TSKEY)value; + pIdx->maxKey.ts = (TSKEY)value; return buf; } From 9bd195d94d2aa6dcdc15cd4f06be19c9698d2043 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sat, 4 Jun 2022 22:33:30 +0800 Subject: [PATCH 22/63] reschedule timeout task --- include/common/tmsg.h | 1 + include/libs/scheduler/scheduler.h | 2 + source/client/inc/clientInt.h | 10 -- source/client/src/clientEnv.c | 6 +- source/client/src/clientHb.c | 101 +++---------- source/client/src/clientImpl.c | 14 -- source/client/src/clientSml.c | 2 + source/libs/catalog/src/ctgDbg.c | 15 ++ source/libs/parser/src/parInsert.c | 2 + source/libs/qworker/inc/qwInt.h | 5 +- source/libs/qworker/src/qwUtil.c | 6 + source/libs/qworker/src/qworker.c | 21 ++- source/libs/scheduler/inc/schedulerInt.h | 4 +- source/libs/scheduler/src/schJob.c | 31 +--- source/libs/scheduler/src/schRemote.c | 86 +---------- source/libs/scheduler/src/schUtil.c | 177 ++++++++++++++++++++++- source/libs/scheduler/src/scheduler.c | 10 ++ 17 files changed, 268 insertions(+), 225 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ee858ad506..dfa641c83e 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1982,6 +1982,7 @@ typedef struct { typedef struct { SClientHbKey connKey; + int64_t clusterId; SQueryHbReqBasic* query; SHashObj* info; // hash } SClientHbReq; diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 5cb6dbd3bc..22706f0953 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -104,6 +104,8 @@ void schedulerAsyncFetchRows(int64_t job, schedulerFetchCallback fp, void* param int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub); +void schedulerStopQueryHb(void *pTrans); + /** * Cancel query job diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index c7b831893e..0000366880 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -58,11 +58,6 @@ enum { typedef struct SAppInstInfo SAppInstInfo; -typedef struct { - void* param; - SClientHbReq* req; -} SHbConnInfo; - typedef struct { char* key; // statistics @@ -72,11 +67,8 @@ typedef struct { int64_t startTime; // ctl SRWLatch lock; // lock is used in serialization - // connection SAppInstInfo* pAppInstInfo; - // info SHashObj* activeInfo; // hash - SHashObj* connInfo; // hash } SAppHbMgr; typedef int32_t (*FHbRspHandle)(SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); @@ -326,8 +318,6 @@ void appHbMgrCleanup(void); int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey); -int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen); - // --- mq void hbMgrInitMqHbRspHandle(); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 9afb989d27..336924f934 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -130,8 +130,12 @@ void destroyTscObj(void *pObj) { SClientHbKey connKey = {.tscRid = pTscObj->id, .connType = pTscObj->connType}; hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); - atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); + int64_t connNum = atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); closeAllRequests(pTscObj->pRequests); + schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter); + if (0 == connNum) { + closeTransporter(pTscObj); + } tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); taosThreadMutexDestroy(&pTscObj->mutex); taosMemoryFreeClear(pTscObj); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 09c3d269c7..a4c109ee17 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -129,9 +129,9 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo } static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { - SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey)); - if (NULL == info) { - tscWarn("fail to get connInfo, may be dropped, refId:%" PRIx64 ", type:%d", pRsp->connKey.tscRid, + SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &pRsp->connKey, sizeof(SClientHbKey)); + if (NULL == pReq) { + tscWarn("pReq to get activeInfo, may be dropped, refId:%" PRIx64 ", type:%d", pRsp->connKey.tscRid, pRsp->connKey.connType); return TSDB_CODE_SUCCESS; } @@ -181,12 +181,11 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { break; } - int64_t *clusterId = (int64_t *)info->param; struct SCatalog *pCatalog = NULL; - int32_t code = catalogGetHandle(*clusterId, &pCatalog); + int32_t code = catalogGetHandle(pReq->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { - tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code)); + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pReq->clusterId, tstrerror(code)); break; } @@ -199,12 +198,11 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { break; } - int64_t *clusterId = (int64_t *)info->param; struct SCatalog *pCatalog = NULL; - int32_t code = catalogGetHandle(*clusterId, &pCatalog); + int32_t code = catalogGetHandle(pReq->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { - tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code)); + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pReq->clusterId, tstrerror(code)); break; } @@ -217,12 +215,11 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { break; } - int64_t *clusterId = (int64_t *)info->param; struct SCatalog *pCatalog = NULL; - int32_t code = catalogGetHandle(*clusterId, &pCatalog); + int32_t code = catalogGetHandle(pReq->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { - tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code)); + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pReq->clusterId, tstrerror(code)); break; } @@ -547,13 +544,10 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { pOneReq = taosArrayPush(pBatchReq->reqs, pOneReq); - SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pOneReq->connKey, sizeof(SClientHbKey)); - if (info) { - code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, info->param, pOneReq); - if (code) { - pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); - continue; - } + code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, &pOneReq->clusterId, pOneReq); + if (code) { + pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); + continue; } //hbClearClientHbReq(pOneReq); @@ -569,23 +563,6 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { return pBatchReq; } -void hbClearReqInfo(SAppHbMgr *pAppHbMgr) { - void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL); - while (pIter != NULL) { - SClientHbReq *pOneReq = pIter; - - tFreeReqKvHash(pOneReq->info); - taosHashClear(pOneReq->info); - - if (pOneReq->query) { - taosArrayDestroy(pOneReq->query->queryDesc); - taosMemoryFreeClear(pOneReq->query); - } - - pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); - } -} - void hbThreadFuncUnexpectedStopped(void) { atomic_store_8(&clientHbMgr.threadStop, 2); } @@ -715,14 +692,6 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { } taosHashSetFreeFp(pAppHbMgr->activeInfo, tFreeClientHbReq); - // init getInfoFunc - pAppHbMgr->connInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); - - if (pAppHbMgr->connInfo == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(pAppHbMgr); - return NULL; - } taosThreadMutexLock(&clientHbMgr.lock); taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr); @@ -745,15 +714,6 @@ void appHbMgrCleanup(void) { taosHashCleanup(pTarget->activeInfo); pTarget->activeInfo = NULL; - pIter = taosHashIterate(pTarget->connInfo, NULL); - while (pIter != NULL) { - SHbConnInfo *info = pIter; - taosMemoryFree(info->param); - pIter = taosHashIterate(pTarget->connInfo, pIter); - } - taosHashCleanup(pTarget->connInfo); - pTarget->connInfo = NULL; - taosMemoryFree(pTarget->key); taosMemoryFree(pTarget); } @@ -791,7 +751,7 @@ void hbMgrCleanUp() { clientHbMgr.appHbMgrs = NULL; } -int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) { +int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, int64_t clusterId) { // init hash in activeinfo void *data = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); if (data != NULL) { @@ -799,17 +759,11 @@ int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo * } SClientHbReq hbReq = {0}; hbReq.connKey = connKey; + hbReq.clusterId = clusterId; //hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); taosHashPut(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey), &hbReq, sizeof(SClientHbReq)); - // init hash - if (info != NULL) { - SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); - info->req = pReq; - taosHashPut(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey), info, sizeof(SHbConnInfo)); - } - atomic_add_fetch_32(&pAppHbMgr->connKeyCnt, 1); return 0; } @@ -819,15 +773,10 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in .tscRid = tscRefId, .connType = connType, }; - SHbConnInfo info = {0}; switch (connType) { case CONN_TYPE__QUERY: { - int64_t *pClusterId = taosMemoryMalloc(sizeof(int64_t)); - *pClusterId = clusterId; - - info.param = pClusterId; - return hbRegisterConnImpl(pAppHbMgr, connKey, &info); + return hbRegisterConnImpl(pAppHbMgr, connKey, clusterId); } case CONN_TYPE__TMQ: { return 0; @@ -844,26 +793,10 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); } - SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey)); - if (info) { - taosMemoryFree(info->param); - taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey)); - } - - if (NULL == pReq || NULL == info) { + if (NULL == pReq) { return; } atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); } -int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *key, void *value, int32_t keyLen, - int32_t valueLen) { - // find req by connection id - SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); - ASSERT(pReq != NULL); - - taosHashPut(pReq->info, key, keyLen, value, valueLen); - - return 0; -} diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6694954fa2..4c68edaff0 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -741,20 +741,6 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { do { destroyRequest(pRequest); pRequest = launchQuery(pTscObj, sql, sqlLen); - if (*sql == 'y') { - SCatalog *pCatalog = NULL; - code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); - SEpSet epset = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - ctgdLaunchAsyncCall(pCatalog, pTscObj->pAppInfo->pTransporter, &epset, pRequest->requestId, false); - break; - } else if (*sql == 'z') { - SCatalog *pCatalog = NULL; - code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); - SEpSet epset = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - ctgdLaunchAsyncCall(pCatalog, pTscObj->pAppInfo->pTransporter, &epset, pRequest->requestId, false); - break; - } - if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) { break; } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index c559ac58f8..5643af746d 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2332,6 +2332,8 @@ static int32_t isSchemalessDb(SSmlHandle* info){ smlBuildInvalidDataMsg(&info->msgBuf, "catalogGetDBCfg error, code:", tstrerror(code)); return code; } + taosArrayDestroy(pInfo.pRetensions); + if (!pInfo.schemaless){ info->pRequest->code = TSDB_CODE_SML_INVALID_DB_CONF; smlBuildInvalidDataMsg(&info->msgBuf, "can not insert into schemaless db:", dbFname); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index cb18e45638..42baa530ce 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -132,6 +132,21 @@ void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { } } + +/* +prepare SQL: +create database db1; +use db1; +create stable st1 (ts timestamp, f1 int) tags(t1 int); +create table tb1 using st1 tags(1); +insert into tb1 values (now, 1); +create qnode on dnode 1; +create user user1 pass "abc"; +create database db2; +grant write on db2.* to user1; +create function udf1 as '/tmp/libudf1.so' outputtype int; +create aggregate function udf2 as '/tmp/libudf2.so' outputtype int; +*/ int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, uint64_t reqId, bool forceUpdate) { int32_t code = 0; SCatalogReq req = {0}; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 422c480397..01a6761b33 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1424,6 +1424,8 @@ int32_t isNotSchemalessDb(SParseContext* pContext, char *dbName){ parserError("catalogGetDBCfg error, code:%s, dbFName:%s", tstrerror(code), dbFname); return code; } + taosArrayDestroy(pInfo.pRetensions); + if (pInfo.schemaless){ parserError("can not insert into schemaless db:%s", dbFname); return TSDB_CODE_SML_INVALID_DB_CONF; diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 3804c114cb..082db6428f 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -34,6 +34,7 @@ extern "C" { #define QW_DEFAULT_SCH_TASK_NUMBER 10000 #define QW_DEFAULT_SHORT_RUN_TIMES 2 #define QW_DEFAULT_HEARTBEAT_MSEC 5000 +#define QW_SCH_TIMEOUT_MSEC 180000 enum { QW_PHASE_PRE_QUERY = 1, @@ -137,7 +138,7 @@ typedef struct SQWTaskCtx { } SQWTaskCtx; typedef struct SQWSchStatus { - int32_t lastAccessTs; // timestamp in second + int64_t hbBrokenTs; // timestamp in msecond SRWLatch hbConnLock; SRpcHandleInfo hbConnInfo; SQueryNodeEpId hbEpId; @@ -354,6 +355,8 @@ int32_t qwOpenRef(void); void qwSetHbParam(int64_t refId, SQWHbParam **pParam); int32_t qwUpdateTimeInQueue(SQWorker *mgmt, int64_t ts, EQueueType type); int64_t qwGetTimeInQueue(SQWorker *mgmt, EQueueType type); +void qwClearExpiredSch(SArray* pExpiredSch); +int32_t qwAcquireScheduler(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch); void qwDbgDumpMgmtInfo(SQWorker *mgmt); int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore); diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 3d0204e355..8bfb80f061 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -535,3 +535,9 @@ int64_t qwGetTimeInQueue(SQWorker *mgmt, EQueueType type) { return -1; } + +void qwClearExpiredSch(SArray* pExpiredSch) { + +} + + diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 8c0366fa0b..44a8fdf7f4 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -21,10 +21,12 @@ int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *re SSchedulerHbRsp rsp = {0}; SQWSchStatus *sch = NULL; - QW_ERR_RET(qwAcquireAddScheduler(mgmt, req->sId, QW_READ, &sch)); + QW_ERR_RET(qwAcquireScheduler(mgmt, req->sId, QW_READ, &sch)); QW_LOCK(QW_WRITE, &sch->hbConnLock); + sch->hbBrokenTs = taosGetTimestampMs(); + if (qwMsg->connInfo.handle == sch->hbConnInfo.handle) { tmsgReleaseHandle(&sch->hbConnInfo, TAOS_CONN_SERVER); sch->hbConnInfo.handle = NULL; @@ -794,6 +796,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { SQWSchStatus *sch = NULL; int32_t taskNum = 0; SQWHbInfo *rspList = NULL; + SArray *pExpiredSch = NULL; int32_t code = 0; qwDbgDumpMgmtInfo(mgmt); @@ -809,8 +812,11 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { } rspList = taosMemoryCalloc(schNum, sizeof(SQWHbInfo)); - if (NULL == rspList) { + pExpiredSch = taosArrayInit(schNum, sizeof(uint64_t)); + if (NULL == rspList || NULL == pExpiredSch) { QW_UNLOCK(QW_READ, &mgmt->schLock); + taosMemoryFree(rspList); + taosArrayDestroy(pExpiredSch); QW_ELOG("calloc %d SQWHbInfo failed", schNum); taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); qwRelease(refId); @@ -820,6 +826,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { void *key = NULL; size_t keyLen = 0; int32_t i = 0; + int64_t currentMs = taosGetTimestampMs(); void *pIter = taosHashIterate(mgmt->schHash, NULL); while (pIter) { @@ -827,6 +834,11 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { if (NULL == sch->hbConnInfo.handle) { uint64_t *sId = taosHashGetKey(pIter, NULL); QW_TLOG("cancel send hb to sch %" PRIx64 " cause of no connection handle", *sId); + + if (sch->hbBrokenTs > 0 && ((currentMs - sch->hbBrokenTs) > QW_SCH_TIMEOUT_MSEC) && taosHashGetSize(sch->tasksHash) <= 0) { + taosArrayPush(pExpiredSch, sId); + } + pIter = taosHashIterate(mgmt->schHash, pIter); continue; } @@ -852,7 +864,12 @@ _return: tFreeSSchedulerHbRsp(&rspList[j].rsp); } + if (taosArrayGetSize(pExpiredSch) > 0) { + qwClearExpiredSch(pExpiredSch); + } + taosMemoryFreeClear(rspList); + taosArrayDestroy(pExpiredSch); taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); qwRelease(refId); diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index c8fee1e532..d961780ae5 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -102,11 +102,11 @@ typedef struct SSchedulerMgmt { uint64_t taskId; // sequential taksId uint64_t sId; // schedulerId SSchedulerCfg cfg; - SRWLatch lock; bool exit; int32_t jobRef; int32_t jobNum; SSchStat stat; + SRWLatch hbLock; SHashObj *hbConnections; } SSchedulerMgmt; @@ -320,6 +320,8 @@ extern SSchedulerMgmt schMgmt; #define SCH_UNLOCK(type, _lock) (SCH_READ == (type) ? taosRUnLockLatch(_lock) : taosWUnLockLatch(_lock)) +void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask); +void schCleanClusterHb(void* pTrans); int32_t schLaunchTask(SSchJob *job, SSchTask *task); int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType); SSchJob *schAcquireJob(int64_t refId); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index be7b930d79..ca90d2fe34 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -126,30 +126,6 @@ _return: SCH_RET(code); } -void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { - if (!pTask->registerdHb) { - return; - } - - SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); - SQueryNodeEpId epId = {0}; - - epId.nodeId = addr->nodeId; - - SEp* pEp = SCH_GET_CUR_EP(addr); - strcpy(epId.ep.fqdn, pEp->fqdn); - epId.ep.port = pEp->port; - - SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, &epId, sizeof(SQueryNodeEpId)); - if (NULL == hb) { - SCH_TASK_ELOG("nodeId %d fqdn %s port %d not in hb connections", epId.nodeId, epId.ep.fqdn, epId.ep.port); - return; - } - - atomic_sub_fetch_64(&hb->taskNum, 1); - - pTask->registerdHb = false; -} void schFreeTask(SSchJob *pJob, SSchTask *pTask) { schDeregisterTaskHb(pJob, pTask); @@ -1484,9 +1460,10 @@ void schFreeJobImpl(void *job) { qDebug("QID:0x%" PRIx64 " job freed, refId:%" PRIx64 ", pointer:%p", queryId, refId, pJob); - atomic_sub_fetch_32(&schMgmt.jobNum, 1); - - schCloseJobRef(); + int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1); + if (jobNum == 0) { + schCloseJobRef(); + } } int32_t schExecJobImpl(void *pTrans, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 1389459604..bf51d8d631 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -648,31 +648,6 @@ _return: SCH_RET(code); } -int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId, bool *exist) { - int32_t code = 0; - SSchHbTrans hb = {0}; - - hb.trans.pTrans = pJob->pTrans; - - SCH_ERR_RET(schMakeHbRpcCtx(pJob, pTask, &hb.rpcCtx)); - - code = taosHashPut(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId), &hb, sizeof(SSchHbTrans)); - if (code) { - schFreeRpcCtx(&hb.rpcCtx); - - if (HASH_NODE_EXIST(code)) { - *exist = true; - return TSDB_CODE_SUCCESS; - } - - qError("taosHashPut hb trans failed, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, epId->ep.port); - SCH_ERR_RET(code); - } - - return TSDB_CODE_SUCCESS; -} - - int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray* taskAction) { SSchedulerHbReq req = {0}; int32_t code = 0; @@ -684,17 +659,20 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray* taskAction) { req.sId = schMgmt.sId; memcpy(&req.epId, nodeEpId, sizeof(SQueryNodeEpId)); + SCH_LOCK(SCH_READ, &schMgmt.hbLock); SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, nodeEpId, sizeof(SQueryNodeEpId)); if (NULL == hb) { - qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", nodeEpId->nodeId, nodeEpId->ep.fqdn, + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); + qError("hb connection no longer exist, nodeId:%d, fqdn:%s, port:%d", nodeEpId->nodeId, nodeEpId->ep.fqdn, nodeEpId->ep.port); - SCH_ERR_RET(code); + return TSDB_CODE_SUCCESS; } SCH_LOCK(SCH_WRITE, &hb->lock); code = schCloneHbRpcCtx(&hb->rpcCtx, &rpcCtx); memcpy(&trans, &hb->trans, sizeof(trans)); SCH_UNLOCK(SCH_WRITE, &hb->lock); + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); SCH_ERR_RET(code); @@ -764,60 +742,6 @@ _return: SCH_RET(code); } - -int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) { - SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); - SQueryNodeEpId epId = {0}; - - epId.nodeId = addr->nodeId; - - SEp* pEp = SCH_GET_CUR_EP(addr); - strcpy(epId.ep.fqdn, pEp->fqdn); - epId.ep.port = pEp->port; - - SSchHbTrans *hb = NULL; - while (true) { - hb = taosHashGet(schMgmt.hbConnections, &epId, sizeof(SQueryNodeEpId)); - if (NULL == hb) { - bool exist = false; - SCH_ERR_RET(schRegisterHbConnection(pJob, pTask, &epId, &exist)); - if (!exist) { - SCH_ERR_RET(schBuildAndSendHbMsg(&epId, NULL)); - } - - continue; - } - - break; - } - - atomic_add_fetch_64(&hb->taskNum, 1); - - pTask->registerdHb = true; - - return TSDB_CODE_SUCCESS; -} - -int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { - int32_t code = 0; - SSchHbTrans *hb = NULL; - - hb = taosHashGet(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); - if (NULL == hb) { - qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, epId->ep.port); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); - } - - SCH_LOCK(SCH_WRITE, &hb->lock); - memcpy(&hb->trans, trans, sizeof(*trans)); - SCH_UNLOCK(SCH_WRITE, &hb->lock); - - qDebug("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, pTrans:%p, pHandle:%p", schMgmt.sId, - epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->pTrans, trans->pHandle); - - return TSDB_CODE_SUCCESS; -} - int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { SSchedulerHbRsp rsp = {0}; SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 81c95ea976..a91dc359f2 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -21,17 +21,187 @@ #include "tref.h" #include "trpc.h" + +void schCleanClusterHb(void* pTrans) { + SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); + + SSchHbTrans *hb = taosHashIterate(schMgmt.hbConnections, NULL); + while (hb) { + if (hb->trans.pTrans == pTrans) { + SQueryNodeEpId* pEpId = taosHashGetKey(hb, NULL); + rpcReleaseHandle(hb->trans.pHandle, TAOS_CONN_CLIENT); + taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); + } + + hb = taosHashIterate(schMgmt.hbConnections, hb); + } + + SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); +} + +int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId) { + int32_t code = 0; + + SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); + SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); + if (NULL == hb) { + SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); + SCH_TASK_ELOG("nodeId %d fqdn %s port %d not in hb connections", epId->nodeId, epId->ep.fqdn, epId->ep.port); + return TSDB_CODE_SUCCESS; + } + + int64_t taskNum = atomic_load_64(&hb->taskNum); + if (taskNum <= 0) { + rpcReleaseHandle(hb->trans.pHandle, TAOS_CONN_CLIENT); + taosHashRemove(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); + } + SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); + + return TSDB_CODE_SUCCESS; +} + + +int32_t schAddHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId, bool *exist) { + int32_t code = 0; + SSchHbTrans hb = {0}; + + hb.trans.pTrans = pJob->pTrans; + hb.taskNum = 1; + + SCH_ERR_RET(schMakeHbRpcCtx(pJob, pTask, &hb.rpcCtx)); + + SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); + code = taosHashPut(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId), &hb, sizeof(SSchHbTrans)); + if (code) { + SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); + schFreeRpcCtx(&hb.rpcCtx); + + if (HASH_NODE_EXIST(code)) { + *exist = true; + return TSDB_CODE_SUCCESS; + } + + qError("taosHashPut hb trans failed, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, epId->ep.port); + SCH_ERR_RET(code); + } + + SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); + + return TSDB_CODE_SUCCESS; +} + +int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *pEpId) { + SSchHbTrans *hb = NULL; + + while (true) { + SCH_LOCK(SCH_READ, &schMgmt.hbLock); + hb = taosHashGet(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); + if (NULL == hb) { + bool exist = false; + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); + SCH_ERR_RET(schAddHbConnection(pJob, pTask, pEpId, &exist)); + if (!exist) { + SCH_RET(schBuildAndSendHbMsg(pEpId, NULL)); + } + + continue; + } + + break; + } + + atomic_add_fetch_64(&hb->taskNum, 1); + + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); + + return TSDB_CODE_SUCCESS; +} + +void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { + if (!pTask->registerdHb) { + return; + } + + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + SQueryNodeEpId epId = {0}; + + epId.nodeId = addr->nodeId; + + SEp* pEp = SCH_GET_CUR_EP(addr); + strcpy(epId.ep.fqdn, pEp->fqdn); + epId.ep.port = pEp->port; + + SCH_LOCK(SCH_READ, &schMgmt.hbLock); + SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, &epId, sizeof(SQueryNodeEpId)); + if (NULL == hb) { + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); + SCH_TASK_WLOG("nodeId %d fqdn %s port %d not in hb connections", epId.nodeId, epId.ep.fqdn, epId.ep.port); + return; + } + + int64_t taskNum = atomic_sub_fetch_64(&hb->taskNum, 1); + if (0 == taskNum) { + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); + schRemoveHbConnection(pJob, pTask, &epId); + } else { + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); + } + + pTask->registerdHb = false; +} + + + +int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) { + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + SQueryNodeEpId epId = {0}; + + epId.nodeId = addr->nodeId; + + SEp* pEp = SCH_GET_CUR_EP(addr); + strcpy(epId.ep.fqdn, pEp->fqdn); + epId.ep.port = pEp->port; + + SCH_ERR_RET(schRegisterHbConnection(pJob, pTask, &epId)); + + pTask->registerdHb = true; + + return TSDB_CODE_SUCCESS; +} + +int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { + int32_t code = 0; + SSchHbTrans *hb = NULL; + + SCH_LOCK(SCH_READ, &schMgmt.hbLock); + hb = taosHashGet(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); + if (NULL == hb) { + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); + qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, epId->ep.port); + SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + SCH_LOCK(SCH_WRITE, &hb->lock); + memcpy(&hb->trans, trans, sizeof(*trans)); + SCH_UNLOCK(SCH_WRITE, &hb->lock); + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); + + qDebug("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, pTrans:%p, pHandle:%p", schMgmt.sId, + epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->pTrans, trans->pHandle); + + return TSDB_CODE_SUCCESS; +} + + void schCloseJobRef(void) { if (!atomic_load_8((int8_t *)&schMgmt.exit)) { return; } - SCH_LOCK(SCH_WRITE, &schMgmt.lock); - if (atomic_load_32(&schMgmt.jobNum) <= 0 && schMgmt.jobRef >= 0) { + if (schMgmt.jobRef >= 0) { taosCloseRef(schMgmt.jobRef); schMgmt.jobRef = -1; } - SCH_UNLOCK(SCH_WRITE, &schMgmt.lock); } uint64_t schGenTaskId(void) { return atomic_add_fetch_64(&schMgmt.taskId, 1); } @@ -88,4 +258,3 @@ void schFreeRpcCtx(SRpcCtx *pCtx) { (*pCtx->freeFunc)(pCtx->brokenVal.val); } - diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 32e00c1b70..b4dc067f4c 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -182,6 +182,14 @@ int32_t scheduleCancelJob(int64_t job) { SCH_RET(code); } +void schedulerStopQueryHb(void *pTrans) { + if (NULL == pTrans) { + return; + } + + schCleanClusterHb(pTrans); +} + void schedulerFreeJob(int64_t job) { SSchJob *pJob = schAcquireJob(job); if (NULL == pJob) { @@ -220,6 +228,7 @@ void schedulerDestroy(void) { } } + SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); if (schMgmt.hbConnections) { void *pIter = taosHashIterate(schMgmt.hbConnections, NULL); while (pIter != NULL) { @@ -230,4 +239,5 @@ void schedulerDestroy(void) { taosHashCleanup(schMgmt.hbConnections); schMgmt.hbConnections = NULL; } + SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); } From 7c9d76a6a653d2f4a6359ecb2e62759f9958693f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 4 Jun 2022 22:33:36 +0800 Subject: [PATCH 23/63] enh: add json index --- include/client/taos.h | 38 ++++++------- source/dnode/vnode/src/meta/metaQuery.c | 4 ++ source/dnode/vnode/src/meta/metaSma.c | 8 +-- source/dnode/vnode/src/meta/metaTable.c | 76 ++++++++++++++++++++++++- source/libs/index/src/index.c | 4 +- source/libs/index/src/indexComm.c | 8 +-- source/libs/index/src/indexFilter.c | 72 ++++++++++++++++++++--- 7 files changed, 170 insertions(+), 40 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index c311e017df..7f317b3a9d 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -103,10 +103,10 @@ typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); typedef struct TAOS_MULTI_BIND { int buffer_type; - void *buffer; + void * buffer; uintptr_t buffer_length; - int32_t *length; - char *is_null; + int32_t * length; + char * is_null; int num; } TAOS_MULTI_BIND; @@ -130,7 +130,7 @@ DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT setConfRet taos_set_config(const char *config); DLL_EXPORT int taos_init(void); -DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); +DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port); DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); @@ -147,17 +147,17 @@ DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name DLL_EXPORT int taos_stmt_get_tag_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields); DLL_EXPORT int taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields); -DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); -DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); -DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); -DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); -DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); -DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx); -DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); +DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); +DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); +DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); +DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); +DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx); +DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); -DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt); +DLL_EXPORT char * taos_stmt_errstr(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt); @@ -179,11 +179,11 @@ DLL_EXPORT bool taos_is_update_query(TAOS_RES *res); DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows); DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData); -DLL_EXPORT int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex); +DLL_EXPORT int * taos_get_column_data_offset(TAOS_RES *res, int columnIndex); DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); DLL_EXPORT void taos_reset_current_db(TAOS *taos); -DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res); +DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res); DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res); DLL_EXPORT const char *taos_get_server_info(TAOS *taos); @@ -204,7 +204,7 @@ DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub); DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress); #endif -DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); +DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); /* --------------------------TMQ INTERFACE------------------------------- */ @@ -229,7 +229,7 @@ DLL_EXPORT tmq_list_t *tmq_list_new(); DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); DLL_EXPORT void tmq_list_destroy(tmq_list_t *); DLL_EXPORT int32_t tmq_list_get_size(const tmq_list_t *); -DLL_EXPORT char **tmq_list_to_c_array(const tmq_list_t *); +DLL_EXPORT char ** tmq_list_to_c_array(const tmq_list_t *); DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen); @@ -240,7 +240,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list); DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq); DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics); -DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout); +DLL_EXPORT TAOS_RES * tmq_consumer_poll(tmq_t *tmq, int64_t timeout); DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets); DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param); @@ -260,7 +260,7 @@ enum tmq_conf_res_t { typedef enum tmq_conf_res_t tmq_conf_res_t; -DLL_EXPORT tmq_conf_t *tmq_conf_new(); +DLL_EXPORT tmq_conf_t * tmq_conf_new(); DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value); DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 21a55d6463..fc7d1c3c92 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -603,6 +603,9 @@ typedef struct { } SIdxCursor; int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { +#ifdef USE_INVERTED_INDEX + return -1; +#else SIdxCursor *pCursor = NULL; int32_t ret = 0, valid = 0; @@ -678,4 +681,5 @@ END: taosMemoryFree(pCursor); return ret; +#endif } diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index fde9d30346..689cd511c4 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -28,9 +28,9 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) { int vLen = 0; const void *pKey = NULL; const void *pVal = NULL; - void *pBuf = NULL; + void * pBuf = NULL; int32_t szBuf = 0; - void *p = NULL; + void * p = NULL; SMetaReader mr = {0}; // validate req @@ -83,8 +83,8 @@ int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) { static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) { STbDbKey tbDbKey; - void *pKey = NULL; - void *pVal = NULL; + void * pKey = NULL; + void * pVal = NULL; int kLen = 0; int vLen = 0; SEncoder coder = {0}; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index b45fccaf29..b6be4d1cf4 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -15,6 +15,7 @@ #include "meta.h" +static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME); static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME); @@ -25,7 +26,7 @@ static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry); static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type); -static int metaUpdateMetaRsp(tb_uid_t uid, char* tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) { +static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) { pMetaRsp->pSchemas = taosMemoryMalloc(pSchema->nCols * sizeof(SSchema)); if (NULL == pMetaRsp->pSchemas) { terrno = TSDB_CODE_VND_OUT_OF_MEMORY; @@ -43,6 +44,75 @@ static int metaUpdateMetaRsp(tb_uid_t uid, char* tbName, SSchemaWrapper *pSchema return 0; } +static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) { +#ifdef USE_INVERTED_INDEX + if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) { + return -1; + } + void * data = pCtbEntry->ctbEntry.pTags; + const char *tagName = pSchema->name; + + tb_uid_t suid = pCtbEntry->ctbEntry.suid; + tb_uid_t tuid = pCtbEntry->uid; + const void *pTagData = pCtbEntry->ctbEntry.pTags; + int32_t nTagData = 0; + + SArray *pTagVals = NULL; + if (tTagToValArray((const STag *)data, &pTagVals) != 0) { + return -1; + } + SIndexMultiTerm *terms = indexMultiTermCreate(); + int16_t nCols = taosArrayGetSize(pTagVals); + for (int i = 0; i < nCols; i++) { + STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); + char type = pTagVal->type; + + char *Key = pTagVal->pKey; + char *key = taosMemoryCalloc(1, strlen(Key) + 2 + strlen(tagName)); + sprintf(key, "%s_%s", tagName, Key); + + int32_t nKey = strlen(key); + SIndexTerm *term = NULL; + + if (type == TSDB_DATA_TYPE_NULL) { + } else if (type == TSDB_DATA_TYPE_NCHAR) { + if (pTagVal->nData > 0) { + char * val = taosMemoryCalloc(1, pTagVal->nData); + int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val); + // printf("val: %s, len: %d", val, len); + + char *tval = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE); + memcpy(tval, (uint16_t *)&len, VARSTR_HEADER_SIZE); + memcpy(tval + VARSTR_HEADER_SIZE, val, len); + type = TSDB_DATA_TYPE_VARCHAR; + term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, tval, len + 2); + } else if (pTagVal->nData == 0) { + char * val = NULL; + int32_t len = 0; + // handle NULL key + } + } else if (type == TSDB_DATA_TYPE_DOUBLE) { + double val = *(double *)(&pTagVal->i64); + int len = 0; + term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len); + } else if (type == TSDB_DATA_TYPE_BOOL) { + int val = *(int *)(&pTagVal->i64); + int len = 0; + term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len); + } + if (term == NULL) { + // handle except later + } else { + indexMultiTermAdd(terms, term); + } + taosMemoryFree(key); + } + tIndexJsonPut(pMeta->pTagIvtIdx, terms, tuid); + indexMultiTermDestroy(terms); +#endif + return -1; +} + int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry me = {0}; int kLen = 0; @@ -341,7 +411,6 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { return 0; } - static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) { void * pVal = NULL; int nVal = 0; @@ -824,6 +893,9 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { } else { // pTagData = pCtbEntry->ctbEntry.pTags; // nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; + pTagData = pCtbEntry->ctbEntry.pTags; + nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; + return metaSaveJsonVarToIdx(pMeta, pCtbEntry, pTagColumn); } // update tag index diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index ba3aea969f..058f3c1915 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -202,7 +202,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { char buf[128] = {0}; ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType}; int32_t sz = indexSerialCacheKey(&key, buf); - indexDebug("suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); + indexDebug("w suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); IndexCache** cache = taosHashGet(index->colObj, buf, sz); assert(*cache != NULL); @@ -330,7 +330,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result char buf[128] = {0}; ICacheKey key = { .suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName), .colType = term->colType}; - indexDebug("suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); + indexDebug("r suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); int32_t sz = indexSerialCacheKey(&key, buf); taosThreadMutexLock(&sIdx->mtx); diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 5310e1c345..74b1773e87 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -402,16 +402,16 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) { break; } case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY - tlen = taosEncodeBinary(NULL, src, strlen(src)); + tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeBinary(dst, src, strlen(src)); + tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); *dst = (char*)*dst - tlen; break; } case TSDB_DATA_TYPE_VARBINARY: - tlen = taosEncodeBinary(NULL, src, strlen(src)); + tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeBinary(dst, src, strlen(src)); + tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); *dst = (char*)*dst - tlen; break; default: diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index b41006b6dd..b272a1cb63 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -162,12 +162,27 @@ static int32_t sifGetValueFromNode(SNode *node, char **value) { return TSDB_CODE_SUCCESS; } +static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { + SOperatorNode *nd = (SOperatorNode *)node; + assert(nodeType(node) == QUERY_NODE_OPERATOR); + SColumnNode *l = (SColumnNode *)nd->pLeft; + SValueNode * r = (SValueNode *)nd->pRight; + + param->colId = l->colId; + param->colValType = l->node.resType.type; + memcpy(param->dbName, l->dbName, sizeof(l->dbName)); + sprintf(param->colName, "%s_%s", l->colName, r->literal); + param->colValType = r->typeData; + return 0; + // memcpy(param->colName, l->colName, sizeof(l->colName)); +} static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { switch (nodeType(node)) { case QUERY_NODE_VALUE: { SValueNode *vn = (SValueNode *)node; SIF_ERR_RET(sifGetValueFromNode(node, ¶m->condValue)); param->colId = -1; + param->colValType = (uint8_t)(vn->node.resType.type); break; } case QUERY_NODE_COLUMN: { @@ -219,17 +234,31 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx indexError("invalid operation node, left: %p, rigth: %p", node->pLeft, node->pRight); SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } + if (node->opType == OP_TYPE_JSON_GET_VALUE || node->opType == OP_TYPE_JSON_CONTAINS) { + return code; + } SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam)); if (NULL == paramList) { SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SIF_ERR_JRET(sifInitParam(node->pLeft, ¶mList[0], ctx)); - if (nParam > 1) { - SIF_ERR_JRET(sifInitParam(node->pRight, ¶mList[1], ctx)); + if (nodeType(node->pLeft) == QUERY_NODE_OPERATOR) { + SNode *interNode = (node->pLeft); + SIF_ERR_JRET(sifInitJsonParam(interNode, ¶mList[0], ctx)); + if (nParam > 1) { + SIF_ERR_JRET(sifInitParam(node->pRight, ¶mList[1], ctx)); + } + paramList[0].colValType = TSDB_DATA_TYPE_JSON; + *params = paramList; + return TSDB_CODE_SUCCESS; + } else { + SIF_ERR_JRET(sifInitParam(node->pLeft, ¶mList[0], ctx)); + if (nParam > 1) { + SIF_ERR_JRET(sifInitParam(node->pRight, ¶mList[1], ctx)); + } + *params = paramList; + return TSDB_CODE_SUCCESS; } - *params = paramList; - return TSDB_CODE_SUCCESS; _return: taosMemoryFree(paramList); SIF_RET(code); @@ -307,18 +336,23 @@ static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) { static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { SIndexMetaArg *arg = &output->arg; #ifdef USE_INVERTED_INDEX - SIndexTerm *tm = indexTermCreate(arg->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName), + SIndexTerm *tm = indexTermCreate(arg->suid, DEFAULT, right->colValType, left->colName, strlen(left->colName), right->condValue, strlen(right->condValue)); if (tm == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } + int ret = 0; EIndexQueryType qtype = 0; SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); indexMultiTermQueryAdd(mtm, tm, qtype); - int ret = indexSearch(arg->metaHandle, mtm, output->result); + if (left->colValType == TSDB_DATA_TYPE_JSON) { + ret = tIndexJsonSearch(arg->metaHandle, mtm, output->result); + } else { + ret = indexSearch(arg->metaHandle, mtm, output->result); + } indexDebug("index filter data size: %d", (int)taosArrayGetSize(output->result)); indexMultiTermQueryDestroy(mtm); return ret; @@ -392,6 +426,14 @@ static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output int id = OP_TYPE_NMATCH; return sifDoIndex(left, right, id, output); } +static int32_t sifJsonContains(SIFParam *left, SIFParam *right, SIFParam *output) { + // return 0 + return 0; +} +static int32_t sifJsonGetValue(SIFParam *left, SIFParam *rigth, SIFParam *output) { + // return 0 + return 0; +} static int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) { // add more except @@ -445,6 +487,14 @@ static int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *sta *status = SFLT_NOT_INDEX; *func = sifNotMatchFunc; return 0; + case OP_TYPE_JSON_CONTAINS: + *status = SFLT_ACCURATE_INDEX; + *func = sifJsonContains; + return 0; + case OP_TYPE_JSON_GET_VALUE: + *status = SFLT_ACCURATE_INDEX; + *func = sifJsonGetValue; + return 0; default: *status = SFLT_NOT_INDEX; *func = sifNullFunc; @@ -460,13 +510,15 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { if (nParam <= 1) { SIF_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } + if (node->opType == OP_TYPE_JSON_GET_VALUE || node->opType == OP_TYPE_JSON_CONTAINS) { + return code; + } SIFParam *params = NULL; - SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); + SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); // ugly code, refactor later output->arg = ctx->arg; - sif_func_t operFn = sifNullFunc; code = sifGetOperFn(node->opType, &operFn, &output->status); if (ctx->noExec) { @@ -573,7 +625,9 @@ EDealRes sifCalcWalker(SNode *node, void *context) { if (QUERY_NODE_LOGIC_CONDITION == nodeType(node)) { return sifWalkLogic(node, ctx); } + if (QUERY_NODE_OPERATOR == nodeType(node)) { + indexInfo("node type for index filter, type: %d", nodeType(node)); return sifWalkOper(node, ctx); } From b997509a759b1c5ebe60312ab6256179ae5145be Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 4 Jun 2022 15:22:20 +0000 Subject: [PATCH 24/63] more work --- source/dnode/vnode/src/inc/tsdb.h | 12 +-- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 2 +- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 87 +++++++++++++++++++++ 3 files changed, 95 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index acf83e4a9c..08cf652d20 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -50,9 +50,8 @@ int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmit int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); /* SMemDataIter */ -void tsdbMemDataIterOpen(SMemDataIter *pIter, TSDBKEY *pKey, int8_t backward); -void tsdbMemDataIterClose(SMemDataIter *pIter); -void tsdbMemDataIterNext(SMemDataIter *pIter); +void tsdbMemDataIterOpen(SMemData *pMemData, TSDBKEY *pKey, int8_t backward, SMemDataIter *pIter); +bool tsdbMemDataIterNext(SMemDataIter *pIter); void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow); // tsdbCommit2.c ============================================================================================== @@ -928,8 +927,11 @@ struct SMemData { }; struct SMemDataIter { - SMemData *pMemData; - TSDBROW row; + SMemData *pMemData; + int8_t backward; + TSDBROW *pRow; + SMemSkipListNode *pNode; // current node + TSDBROW row; }; #endif diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 38e4795a83..afd5cd72ee 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -137,7 +137,7 @@ static int32_t tsdbCommitToFile(SCommitH *pCHandle, int32_t fid) { // check if there are data in the time range for (; iMemData < nMemData; iMemData++) { SMemData *pMemData = (SMemData *)taosArrayGetP(pCHandle->pMemTable->aMemData, iMemData); - tsdbMemDataIterOpen(&iter, &(TSDBKEY){.ts = fidSKey, .version = 0}, 0); + tsdbMemDataIterOpen(pMemData, &(TSDBKEY){.ts = fidSKey, .version = 0}, 0, &iter); tsdbMemDataIterGet(&iter, &pRow); if (pRow->tsRow.ts >= fidSKey && pRow->tsRow.ts <= fidEKey) { diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index dee025cae6..e581e9fe3d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -42,6 +42,7 @@ static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow); static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl); static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, int64_t version, SVSubmitBlk *pSubmitBlk); +static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags); // SMemTable ============================================== int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) { @@ -160,6 +161,92 @@ _err: return code; } +void tsdbMemDataIterOpen(SMemData *pMemData, TSDBKEY *pKey, int8_t backward, SMemDataIter *pIter) { + SMemSkipListNode *pos[SL_MAX_LEVEL]; + + pIter->pMemData = pMemData; + pIter->backward = backward; + pIter->pRow = NULL; + if (pKey == NULL) { + // create from head or tail + if (backward) { + pIter->pNode = SL_NODE_BACKWARD(pMemData->sl.pTail, 0); + } else { + pIter->pNode = SL_NODE_FORWARD(pMemData->sl.pHead, 0); + } + } else { + // create from a key + if (backward) { + memDataMovePosTo(pMemData, pos, pKey, SL_MOVE_BACKWARD); + pIter->pNode = SL_NODE_BACKWARD(pos[0], 0); + } else { + memDataMovePosTo(pMemData, pos, pKey, 0); + pIter->pNode = SL_NODE_FORWARD(pos[0], 0); + } + } +} + +bool tsdbMemDataIterNext(SMemDataIter *pIter) { + SMemSkipListNode *pHead = pIter->pMemData->sl.pHead; + SMemSkipListNode *pTail = pIter->pMemData->sl.pTail; + + pIter->pRow = NULL; + if (pIter->backward) { + ASSERT(pIter->pNode != pTail); + + if (pIter->pNode == pHead) { + return false; + } + + pIter->pNode = SL_NODE_BACKWARD(pIter->pNode, 0); + if (pIter->pNode == pHead) { + return false; + } + } else { + ASSERT(pIter->pNode != pHead); + + if (pIter->pNode == pTail) { + return false; + } + + pIter->pNode = SL_NODE_FORWARD(pIter->pNode, 0); + if (pIter->pNode == pTail) { + return false; + } + } + + return true; +} + +void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow) { + if (pIter->pRow) { + *ppRow = pIter->pRow; + } else { + SMemSkipListNode *pHead = pIter->pMemData->sl.pHead; + SMemSkipListNode *pTail = pIter->pMemData->sl.pTail; + + if (pIter->backward) { + ASSERT(pIter->pNode != pTail); + + if (pIter->pNode == pHead) { + *ppRow = NULL; + } else { + tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row); + *ppRow = &pIter->row; + } + } else { + ASSERT(pIter->pNode != pHead); + + if (pIter->pNode == pTail) { + *ppRow = NULL; + } else { + tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row); + *ppRow = &pIter->row; + } + } + } +} + static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData) { int32_t code = 0; int32_t idx = 0; From 338f6eea718ad6fea4114b599fcbd0b3cab788b7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 5 Jun 2022 00:47:03 +0800 Subject: [PATCH 25/63] fix(query): fix invalid free in exchange. --- source/libs/executor/src/executorimpl.c | 43 +++++++++++-------------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0cfeecb28c..1a73e9f3ac 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2418,6 +2418,9 @@ int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) pRsp->compLen = htonl(pRsp->compLen); pRsp->numOfCols = htonl(pRsp->numOfCols); pRsp->useconds = htobe64(pRsp->useconds); + + ASSERT(pSourceDataInfo->pRsp != NULL); + qDebug("fetch rsp received, index:%d, rows:%d", pSourceDataInfo->index, pRsp->numOfRows); } else { pSourceDataInfo->code = code; } @@ -2466,6 +2469,8 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex); SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); + ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY); + qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, sourceIndex, totalSources); @@ -2643,7 +2648,6 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx int32_t completed = 0; for (int32_t i = 0; i < totalSources; ++i) { SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, i); - if (pDataInfo->status == EX_SOURCE_DATA_EXHAUSTED) { completed += 1; continue; @@ -2653,6 +2657,11 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx continue; } + if (pDataInfo->code != TSDB_CODE_SUCCESS) { + code = pDataInfo->code; + goto _error; + } + SRetrieveTableRsp* pRsp = pDataInfo->pRsp; SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, i); @@ -2690,6 +2699,8 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx pLoadInfo->totalSize); } + taosMemoryFreeClear(pDataInfo->pRsp); + if (pDataInfo->status != EX_SOURCE_DATA_EXHAUSTED) { pDataInfo->status = EX_SOURCE_DATA_NOT_READY; code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); @@ -2699,7 +2710,6 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx } } - taosMemoryFreeClear(pDataInfo->pRsp); return pExchangeInfo->pResult; } @@ -2719,26 +2729,9 @@ static SSDataBlock* concurrentlyLoadRemoteData(SOperatorInfo* pOperator) { if (pOperator->status == OP_RES_TO_RETURN) { return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); + } else { + ASSERT(0); } - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - int64_t startTs = taosGetTimestampUs(); - - // Asynchronously send all fetch requests to all sources. - for (int32_t i = 0; i < totalSources; ++i) { - int32_t code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); - if (code != TSDB_CODE_SUCCESS) { - return NULL; - } - } - - int64_t endTs = taosGetTimestampUs(); - qDebug("%s send all fetch request to %" PRIzu " sources completed, elapsed:%" PRId64, GET_TASKID(pTaskInfo), - totalSources, endTs - startTs); - - tsem_wait(&pExchangeInfo->ready); - pOperator->status = OP_RES_TO_RETURN; - return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); } static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { @@ -2758,10 +2751,11 @@ static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { } int64_t endTs = taosGetTimestampUs(); - qDebug("%s send all fetch request to %" PRIzu " sources completed, elapsed:%" PRId64, GET_TASKID(pTaskInfo), + qDebug("%s send all fetch requests to %" PRIzu " sources completed, elapsed:%" PRId64, GET_TASKID(pTaskInfo), totalSources, endTs - startTs); tsem_wait(&pExchangeInfo->ready); + pOperator->status = OP_RES_TO_RETURN; pOperator->cost.openCost = taosGetTimestampUs() - startTs; return TSDB_CODE_SUCCESS; @@ -2875,7 +2869,8 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { if (pExchangeInfo->seqLoadData) { return seqLoadRemoteData(pOperator); } else { - return concurrentlyLoadRemoteData(pOperator); + return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); +// return concurrentlyLoadRemoteData(pOperator); } } @@ -2927,7 +2922,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, const SNodeList* p } pInfo->pResult = pBlock; - pInfo->seqLoadData = true; + pInfo->seqLoadData = false; tsem_init(&pInfo->ready, 0, 0); From 8a3594dd8c1b11d1308160147a917445c6bf9ee7 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 5 Jun 2022 00:49:20 +0800 Subject: [PATCH 26/63] feat: add vgroup and epset in tsma info --- include/common/tmsg.h | 7 +++++++ source/common/src/tmsg.c | 25 ++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 507ce7ebaf..202aeaf69e 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2296,6 +2296,11 @@ typedef struct { int32_t tSerializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); +typedef struct { + int32_t vgId; + SEpSet epSet; +} SVgEpSet; + typedef struct { int8_t version; // for compatibility(default 0) int8_t intervalUnit; // MACRO: TIME_UNIT_XXX @@ -2305,6 +2310,7 @@ typedef struct { char indexName[TSDB_INDEX_NAME_LEN]; int32_t exprLen; int32_t tagsFilterLen; + int32_t numOfVgroups; int64_t indexUid; tb_uid_t tableUid; // super/child/common table uid int64_t interval; @@ -2312,6 +2318,7 @@ typedef struct { int64_t sliding; char* expr; // sma expression char* tagsFilter; + SVgEpSet vgEpSet[]; } STSma; // Time-range-wise SMA typedef STSma SVCreateTSmaReq; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 477eff0004..9c9f33ac96 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3661,6 +3661,7 @@ int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) { if (tEncodeCStr(pCoder, pSma->indexName) < 0) return -1; if (tEncodeI32(pCoder, pSma->exprLen) < 0) return -1; if (tEncodeI32(pCoder, pSma->tagsFilterLen) < 0) return -1; + if (tEncodeI32(pCoder, pSma->numOfVgroups) < 0) return -1; if (tEncodeI64(pCoder, pSma->indexUid) < 0) return -1; if (tEncodeI64(pCoder, pSma->tableUid) < 0) return -1; if (tEncodeI64(pCoder, pSma->interval) < 0) return -1; @@ -3672,7 +3673,17 @@ int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) { if (pSma->tagsFilterLen > 0) { if (tEncodeCStr(pCoder, pSma->tagsFilter) < 0) return -1; } - + for (int32_t v = 0; v < pSma->numOfVgroups; ++v) { + if (tEncodeI32(pCoder, pSma->vgEpSet[v].vgId) < 0) return -1; + if (tEncodeI8(pCoder, pSma->vgEpSet[v].epSet.inUse) < 0) return -1; + int8_t numOfEps = pSma->vgEpSet[v].epSet.numOfEps; + if (tEncodeI8(pCoder, numOfEps) < 0) return -1; + for (int32_t n = 0; n < numOfEps; ++n) { + const SEp *pEp = &pSma->vgEpSet[v].epSet.eps[n]; + if (tEncodeCStr(pCoder, pEp->fqdn) < 0) return -1; + if (tEncodeU16(pCoder, pEp->port) < 0) return -1; + } + } return 0; } @@ -3685,6 +3696,7 @@ int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) { if (tDecodeCStrTo(pCoder, pSma->indexName) < 0) return -1; if (tDecodeI32(pCoder, &pSma->exprLen) < 0) return -1; if (tDecodeI32(pCoder, &pSma->tagsFilterLen) < 0) return -1; + if (tDecodeI32(pCoder, &pSma->numOfVgroups) < 0) return -1; if (tDecodeI64(pCoder, &pSma->indexUid) < 0) return -1; if (tDecodeI64(pCoder, &pSma->tableUid) < 0) return -1; if (tDecodeI64(pCoder, &pSma->interval) < 0) return -1; @@ -3700,6 +3712,17 @@ int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) { } else { pSma->tagsFilter = NULL; } + for (int32_t v = 0; v < pSma->numOfVgroups; ++v) { + if (tDecodeI32(pCoder, &pSma->vgEpSet[v].vgId) < 0) return -1; + if (tDecodeI8(pCoder, &pSma->vgEpSet[v].epSet.inUse) < 0) return -1; + if (tDecodeI8(pCoder, &pSma->vgEpSet[v].epSet.numOfEps) < 0) return -1; + int8_t numOfEps = pSma->vgEpSet[v].epSet.numOfEps; + for (int32_t n = 0; n < numOfEps; ++n) { + SEp *pEp = &pSma->vgEpSet[v].epSet.eps[n]; + if (tDecodeCStrTo(pCoder, pEp->fqdn) < 0) return -1; + if (tDecodeU16(pCoder, &pEp->port) < 0) return -1; + } + } return 0; } From 31f10572c4e8f15f11d8502a99896c4565457cb4 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sun, 5 Jun 2022 10:18:03 +0800 Subject: [PATCH 27/63] remove rpc close --- source/client/src/clientEnv.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 336924f934..be0a41ba40 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -134,7 +134,8 @@ void destroyTscObj(void *pObj) { closeAllRequests(pTscObj->pRequests); schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter); if (0 == connNum) { - closeTransporter(pTscObj); + // TODO + //closeTransporter(pTscObj); } tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); taosThreadMutexDestroy(&pTscObj->mutex); From fe27ee265428b7e03d24b8e75424fe6e4cd57fe8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 5 Jun 2022 03:47:41 +0000 Subject: [PATCH 28/63] more work --- source/dnode/vnode/src/inc/tsdb.h | 48 +++---- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 144 +++++++++++++++++++- source/dnode/vnode/src/tsdb/tsdbFS.c | 4 +- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 3 + 4 files changed, 158 insertions(+), 41 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 08cf652d20..02b9e4fcff 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -310,33 +310,13 @@ typedef struct { SBlock blocks[]; } SBlockInfo; -#ifdef TD_REFACTOR_3 -typedef struct { - int16_t colId; - uint16_t bitmap : 1; // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows - uint16_t reserve : 15; - int32_t len; - uint32_t type : 8; - uint32_t offset : 24; - int64_t sum; - int64_t max; - int64_t min; - int16_t maxIndex; - int16_t minIndex; - int16_t numOfNull; - uint8_t offsetH; - char padding[1]; -} SBlockCol; -#else typedef struct { int16_t colId; uint16_t type : 6; uint16_t blen : 10; // 0 no bitmap if all rows are NORM, > 0 bitmap length uint32_t len; // data length + bitmap length uint32_t offset; -} SBlockColV0; - -#define SBlockCol SBlockColV0 // latest SBlockCol definition +} SBlockCol; typedef struct { int16_t colId; @@ -346,11 +326,7 @@ typedef struct { int64_t sum; int64_t max; int64_t min; -} SAggrBlkColV0; - -#define SAggrBlkCol SAggrBlkColV0 // latest SAggrBlkCol definition - -#endif +} SAggrBlkCol; // Code here just for back-ward compatibility static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) { @@ -411,8 +387,7 @@ struct SReadH { #define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf) #define TSDB_READ_EXBUF(rh) ((rh)->pExBuf) -#define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) \ - (sizeof(SBlockData) + sizeof(SBlockColV##blkVer) * (ncols) + sizeof(TSCKSUM)) +#define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) (sizeof(SBlockData) + sizeof(SBlockCol) * (ncols) + sizeof(TSCKSUM)) static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) { switch (blkVer) { @@ -422,7 +397,7 @@ static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) { } } -#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkColV##blkVer) * (ncols) + sizeof(TSCKSUM)) +#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkCol) * (ncols) + sizeof(TSCKSUM)) static FORCE_INLINE size_t tsdbBlockAggrSize(int nCols, uint32_t blkVer) { switch (blkVer) { @@ -520,11 +495,11 @@ static FORCE_INLINE void *taosTZfree(void *ptr) { void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn); -static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { +static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t minutes, int8_t precision) { if (key < 0) { - return (int)((key + 1) / tsTickPerMin[precision] / days - 1); + return (int)((key + 1) / tsTickPerMin[precision] / minutes - 1); } else { - return (int)((key / tsTickPerMin[precision] / days)); + return (int)((key / tsTickPerMin[precision] / minutes)); } } @@ -877,12 +852,21 @@ struct SDelOp { SDelOp *pNext; }; +typedef struct { + tb_uid_t suid; + tb_uid_t uid; + int64_t version; + TSKEY sKey; + TSKEY eKey; +} SDelInfo; + struct SMemTable { STsdb *pTsdb; int32_t nRef; TSDBKEY minKey; TSDBKEY maxKey; int64_t nRows; + int64_t nDelOp; SArray *aSkmInfo; SArray *aMemData; }; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index afd5cd72ee..4948d8a0b1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -17,13 +17,22 @@ typedef struct { SMemTable *pMemTable; + int32_t minutes; + int8_t precision; + int32_t sfid; + int32_t efid; SReadH readh; + SDFileSet wSet; + SArray *aDelInfo; SArray *aBlkIdx; + SArray *aSupBlk; + SArray *aSubBlk; } SCommitH; static int32_t tsdbStartCommit(SCommitH *pCHandle, STsdb *pTsdb); static int32_t tsdbEndCommit(SCommitH *pCHandle); static int32_t tsdbCommitToFile(SCommitH *pCHandle, int32_t fid); +static int32_t tsdbCommitDelete(SCommitH *pCHandle); int32_t tsdbBegin2(STsdb *pTsdb) { int32_t code = 0; @@ -50,15 +59,18 @@ int32_t tsdbCommit2(STsdb *pTsdb) { } // commit - int32_t sfid; // todo - int32_t efid; // todo - for (int32_t fid = sfid; fid <= efid; fid++) { + for (int32_t fid = ch.sfid; fid <= ch.efid; fid++) { code = tsdbCommitToFile(&ch, fid); if (code) { goto _err; } } + code = tsdbCommitDelete(&ch); + if (code) { + goto _err; + } + // end commit code = tsdbEndCommit(&ch); if (code) { @@ -74,18 +86,77 @@ _err: } static int32_t tsdbStartCommit(SCommitH *pCHandle, STsdb *pTsdb) { - int32_t code = 0; + int32_t code = 0; + SMemTable *pMemTable = (SMemTable *)pTsdb->mem; + tsdbInfo("vgId:%d start to commit", TD_VID(pTsdb->pVnode)); + + // switch to commit ASSERT(pTsdb->imem == NULL && pTsdb->mem); pTsdb->imem = pTsdb->mem; pTsdb->mem = NULL; - // TODO + + // open handle + pCHandle->pMemTable = pMemTable; + pCHandle->minutes = pTsdb->keepCfg.days; + pCHandle->precision = pTsdb->keepCfg.precision; + pCHandle->sfid = TSDB_KEY_FID(pMemTable->minKey.ts, pCHandle->minutes, pCHandle->precision); + pCHandle->efid = TSDB_KEY_FID(pMemTable->maxKey.ts, pCHandle->minutes, pCHandle->precision); + + code = tsdbInitReadH(&pCHandle->readh, pTsdb); + if (code) { + goto _err; + } + pCHandle->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx)); + if (pCHandle->aBlkIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pCHandle->aSupBlk = taosArrayInit(1024, sizeof(SBlock)); + if (pCHandle->aSupBlk == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pCHandle->aSubBlk = taosArrayInit(1024, sizeof(SBlock)); + if (pCHandle->aSubBlk == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + // start FS transaction + tsdbStartFSTxn(pTsdb, 0, 0); + + return code; + +_err: return code; } static int32_t tsdbEndCommit(SCommitH *pCHandle) { - int32_t code = 0; - // TODO + int32_t code = 0; + STsdb *pTsdb = pCHandle->pMemTable->pTsdb; + SMemTable *pMemTable = (SMemTable *)pTsdb->imem; + + // end transaction + code = tsdbEndFSTxn(pTsdb); + if (code) { + goto _err; + } + + // close handle + taosArrayClear(pCHandle->aSubBlk); + taosArrayClear(pCHandle->aSupBlk); + taosArrayClear(pCHandle->aBlkIdx); + tsdbDestroyReadH(&pCHandle->readh); + + // destroy memtable (todo: unref it) + pTsdb->imem = NULL; + tsdbMemTableDestroy2(pMemTable); + + tsdbInfo("vgId:%d commit over", TD_VID(pTsdb->pVnode)); + return code; + +_err: return code; } @@ -193,6 +264,65 @@ static int32_t tsdbCommitToFile(SCommitH *pCHandle, int32_t fid) { return code; +_err: + return code; +} + +static int32_t delInfoCmprFn(const void *p1, const void *p2) { + SDelInfo *pDelInfo1 = (SDelInfo *)p1; + SDelInfo *pDelInfo2 = (SDelInfo *)p2; + + if (pDelInfo1->suid < pDelInfo2->suid) { + return -1; + } else if (pDelInfo1->suid > pDelInfo2->suid) { + return 1; + } + + if (pDelInfo1->uid < pDelInfo2->uid) { + return -1; + } else if (pDelInfo1->uid > pDelInfo2->uid) { + return 1; + } + + if (pDelInfo1->version < pDelInfo2->version) { + return -1; + } else if (pDelInfo1->version > pDelInfo2->version) { + return 1; + } + + return 0; +} +static int32_t tsdbCommitDelete(SCommitH *pCHandle) { + int32_t code = 0; + SDelInfo delInfo; + SMemData *pMemData; + + if (pCHandle->pMemTable->nDelOp == 0) goto _exit; + + // load del array (todo) + + // loop to append SDelInfo + for (int32_t iMemData = 0; iMemData < taosArrayGetSize(pCHandle->pMemTable->aMemData); iMemData++) { + pMemData = (SMemData *)taosArrayGetP(pCHandle->pMemTable->aMemData, iMemData); + + for (SDelOp *pDelOp = pMemData->delOpHead; pDelOp; pDelOp = pDelOp->pNext) { + delInfo = (SDelInfo){.suid = pMemData->suid, + .uid = pMemData->uid, + .version = pDelOp->version, + .sKey = pDelOp->sKey, + .eKey = pDelOp->eKey}; + if (taosArrayPush(pCHandle->aDelInfo, &delInfo) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + } + + taosArraySort(pCHandle->aDelInfo, delInfoCmprFn); + +_exit: + return code; + _err: return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index c0ca2f9594..584a1ec49c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -37,11 +37,11 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired); // static int tsdbProcessExpiredFS(STsdb *pRepo); // static int tsdbCreateMeta(STsdb *pRepo); -static void tsdbGetRootDir(int repoid, const char* dir, char dirName[]) { +static void tsdbGetRootDir(int repoid, const char *dir, char dirName[]) { snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, dir); } -static void tsdbGetDataDir(int repoid, const char* dir, char dirName[]) { +static void tsdbGetDataDir(int repoid, const char *dir, char dirName[]) { snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, dir); } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index e581e9fe3d..cf8557bba3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -59,6 +59,7 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) { pMemTable->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX}; pMemTable->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN}; pMemTable->nRows = 0; + pMemTable->nDelOp = 0; pMemTable->aMemData = taosArrayInit(512, sizeof(SMemData *)); if (pMemTable->aMemData == NULL) { taosMemoryFree(pMemTable); @@ -149,6 +150,8 @@ int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_ui // update the state of pMemTable, pMemData, last and lastrow (todo) } + pMemTable->nDelOp++; + tsdbDebug("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64 " since %s", TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code)); From 2c133b4a279f8a45eb08e85474d977bad5b62dce Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 5 Jun 2022 12:28:12 +0800 Subject: [PATCH 29/63] notify if server receice release req --- source/libs/transport/src/transSvr.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 608fd00b2c..82d2b7a6db 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -180,6 +180,12 @@ static bool addHandleToAcceptloop(void* arg); if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \ return; \ } \ + if (conn->regArg.init) { \ + tTrace("server conn %p release, notify server app", conn); \ + STrans* pTransInst = conn->pTransInst; \ + (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); \ + memset(&conn->regArg, 0, sizeof(conn->regArg)); \ + } \ uvStartSendRespInternal(srvMsg); \ return; \ } \ From 9e1cd759f32c8e59390b6cc60a14a2bba91fb36b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 5 Jun 2022 13:50:31 +0800 Subject: [PATCH 30/63] feat: add sort/group logic for json --- include/common/tdatablock.h | 12 +-- source/libs/scalar/src/sclvector.c | 93 ++++++++++--------- .../libs/scalar/test/scalar/scalarTests.cpp | 4 +- tests/system-test/2-query/json_tag.py | 8 +- 4 files changed, 57 insertions(+), 60 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index cce763cafe..dcc074e8bd 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -71,20 +71,14 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet); #define colDataGetData(p1_, r_) \ ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) : colDataGetNumData(p1_, r_)) -static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { - if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON) { - if (colDataIsNull_var(pColumnInfoData, row)) { - return true; - } - char* data = colDataGetVarData(pColumnInfoData, row); - return (*data == TSDB_DATA_TYPE_NULL); - } +#define IS_JSON_NULL(type,data) ((type) == TSDB_DATA_TYPE_JSON && *(data) == TSDB_DATA_TYPE_NULL) +static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { if (!pColumnInfoData->hasNull) { return false; } - if (pColumnInfoData->info.type == TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) { + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { return colDataIsNull_var(pColumnInfoData, row); } else { if (pColumnInfoData->nullbitmap == NULL) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index e844b3cdb6..b931badab9 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -29,6 +29,12 @@ #define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData)) #define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData)) +#define IS_NULL colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i) \ + || IS_JSON_NULL(pLeft->columnData->info.type, colDataGetVarData(pLeft->columnData, i)) \ + || IS_JSON_NULL(pRight->columnData->info.type, colDataGetVarData(pRight->columnData, i)) + +#define IS_HELPER_NULL(col,i) colDataIsNull_s(col, i) || IS_JSON_NULL(col->info.type, colDataGetVarData(col, i)) + void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){ switch (outType) { case TSDB_DATA_TYPE_BOOL: { @@ -867,11 +873,11 @@ static void vectorMathAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig double *output = (double *)pOutputCol->pData; - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -887,11 +893,11 @@ static void vectorMathBigintAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData int64_t *output = (int64_t *)pOutputCol->pData; - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -982,7 +988,7 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + if (IS_NULL) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1000,7 +1006,7 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + if (IS_NULL){ colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1024,11 +1030,11 @@ static void vectorMathSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig double *output = (double *)pOutputCol->pData; - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1044,11 +1050,11 @@ static void vectorMathBigintSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData int64_t *output = (int64_t *)pOutputCol->pData; - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1077,7 +1083,7 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + if (IS_NULL) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1095,7 +1101,7 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + if (IS_NULL) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1119,11 +1125,11 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData* double *output = (double *)pOutputCol->pData; - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1149,7 +1155,7 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam double *output = (double *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + if (IS_NULL) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1182,7 +1188,7 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p double *output = (double *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i) || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { //divide by 0 check + if (IS_NULL || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { //divide by 0 check colDataAppendNULL(pOutputCol, i); continue; } @@ -1190,11 +1196,11 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p / getVectorDoubleValueFnRight(RIGHT_COL, i); } } else if (pLeft->numOfRows == 1) { - if (colDataIsNull_s(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); } else { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pRightCol, i) || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { // divide by 0 check + if (IS_HELPER_NULL(pRightCol, i) || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { // divide by 0 check colDataAppendNULL(pOutputCol, i); continue; } @@ -1203,11 +1209,11 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p } } } else if (pRight->numOfRows == 1) { - if (colDataIsNull_s(pRightCol, 0) || (getVectorDoubleValueFnRight(RIGHT_COL, 0) == 0)) { // Set pLeft->numOfRows NULL value (divde by 0 check) + if (IS_HELPER_NULL(pRightCol, 0) || (getVectorDoubleValueFnRight(RIGHT_COL, 0) == 0)) { // Set pLeft->numOfRows NULL value (divde by 0 check) colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); } else { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; } @@ -1240,7 +1246,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i) || colDataIsNull_s(pRightCol, i)) { + if (IS_NULL) { colDataAppendNULL(pOutputCol, i); continue; } @@ -1256,11 +1262,11 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam } } else if (pLeft->numOfRows == 1) { double lx = getVectorDoubleValueFnLeft(LEFT_COL, 0); - if (colDataIsNull_s(pLeftCol, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pLeftCol, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); } else { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pRightCol, i)) { + if (IS_HELPER_NULL(pRightCol, i)) { colDataAppendNULL(pOutputCol, i); continue; } @@ -1276,11 +1282,11 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam } } else if (pRight->numOfRows == 1) { double rx = getVectorDoubleValueFnRight(RIGHT_COL, 0); - if (colDataIsNull_s(pRightCol, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pRightCol, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); } else { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; } @@ -1315,7 +1321,7 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO double *output = (double *)pOutputCol->pData; for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; } @@ -1331,7 +1337,7 @@ void vectorAssign(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, pOut->numOfRows = pLeft->numOfRows; - if (colDataIsNull_s(pRight->columnData, 0)) { + if (IS_HELPER_NULL(pRight->columnData, 0)) { for (int32_t i = 0; i < pOut->numOfRows; ++i) { colDataAppend(pOutputCol, i, NULL, true); } @@ -1399,11 +1405,11 @@ static void vectorBitAndHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRigh double *output = (double *)pOutputCol->pData; - if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1429,7 +1435,7 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int64_t *output = (int64_t *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + if (IS_NULL) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1451,12 +1457,12 @@ static void vectorBitOrHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRight int64_t *output = (int64_t *)pOutputCol->pData; - if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { int64_t rx = getVectorBigintValueFnRight(RIGHT_COL, 0); for (; i >= 0 && i < numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1482,7 +1488,7 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int64_t *output = (int64_t *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + if (IS_NULL) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1507,7 +1513,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * if (pRight->pHashFilter != NULL) { for (; i >= 0 && i < pLeft->numOfRows; i += step) { - if (colDataIsNull_s(pLeft->columnData, i)) { + if (IS_HELPER_NULL(pLeft->columnData, i)) { bool res = false; colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); continue; @@ -1522,7 +1528,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + if (IS_HELPER_NULL(pLeft->columnData, i) || IS_HELPER_NULL(pRight->columnData, i)) { bool res = false; colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); continue; // TODO set null or ignore @@ -1536,8 +1542,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * bool isJsonnull = false; convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); if(isJsonnull){ - colDataAppendNULL(pOut->columnData, i); - continue; // TODO set null or ignore + ASSERT(0); } bool res = filterDoCompare(fp, optr, pLeftData, pRightData); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); @@ -1545,7 +1550,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * } else if (pRight->numOfRows == 1) { ASSERT(pLeft->pHashFilter == NULL); for (; i >= 0 && i < pLeft->numOfRows; i += step) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, 0)) { + if (IS_HELPER_NULL(pLeft->columnData, i) || IS_HELPER_NULL(pRight->columnData, 0)) { bool res = false; colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); continue; @@ -1558,15 +1563,14 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * bool isJsonnull = false; convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); if(isJsonnull){ - colDataAppendNULL(pOut->columnData, i); - continue; // TODO set null or ignore + ASSERT(0); } bool res = filterDoCompare(fp, optr, pLeftData, pRightData); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } } else if (pLeft->numOfRows == 1) { for (; i >= 0 && i < pRight->numOfRows; i += step) { - if (colDataIsNull_s(pRight->columnData, i) || colDataIsNull_s(pLeft->columnData, 0)) { + if (IS_HELPER_NULL(pRight->columnData, i) || IS_HELPER_NULL(pLeft->columnData, 0)) { bool res = false; colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); continue; @@ -1579,8 +1583,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * bool isJsonnull = false; convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); if(isJsonnull){ - colDataAppendNULL(pOut->columnData, i); - continue; // TODO set null or ignore + ASSERT(0); } bool res = filterDoCompare(fp, optr, pLeftData, pRightData); colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); @@ -1668,7 +1671,7 @@ void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) { - int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0; + int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 1 : 0; colDataAppendInt8(pOut->columnData, i, &v); } pOut->numOfRows = pLeft->numOfRows; @@ -1676,7 +1679,7 @@ void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) { - int8_t v = colDataIsNull_s(pLeft->columnData, i)? 0:1; + int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 0 : 1; colDataAppendInt8(pOut->columnData, i, &v); } pOut->numOfRows = pLeft->numOfRows; diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 8a29462a2b..c71521535a 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -1227,7 +1227,7 @@ TEST(columnTest, json_column_logic_op) { printf("--------------------json null---------------------\n"); key = "k3"; // (null is true) return NULL, so use DBL_MAX represent NULL - double eRes2[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false}; + double eRes2[len+len1] = {false, false, false, false, false, false, true, false, false, false, false, false, false}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]); } @@ -1279,7 +1279,7 @@ TEST(columnTest, json_column_logic_op) { printf("---------------------json not exist--------------------\n"); key = "k10"; // (NULL is true) return NULL, so use DBL_MAX represent NULL - double eRes10[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false}; + double eRes10[len+len1] = {false, false, false, false, false, false, true, false, false, false, false, false, false}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i]); } diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 8f67dd7d42..33f35eabbe 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -137,9 +137,9 @@ class TDTestCase: tdSql.checkRows(9) tdSql.query("select jtag from jsons1") tdSql.checkRows(13) - # tdSql.query("select jtag from jsons1 where jtag is null") + tdSql.query("select jtag from jsons1 where jtag is null") # tdSql.checkRows(5) - # tdSql.query("select jtag from jsons1 where jtag is not null") + tdSql.query("select jtag from jsons1 where jtag is not null") # tdSql.checkRows(8) # test jtag is NULL @@ -260,9 +260,9 @@ class TDTestCase: # tdSql.checkRows(1) # # # where json is null - # tdSql.query("select * from jsons1 where jtag is null") + tdSql.query("select * from jsons1 where jtag is null") # tdSql.checkRows(1) - # tdSql.query("select * from jsons1 where jtag is not null") + tdSql.query("select * from jsons1 where jtag is not null") # tdSql.checkRows(8) # # # where json key is null From 84fd45c3bbfc408421e396acfa4db9b83b543618 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 5 Jun 2022 14:18:40 +0800 Subject: [PATCH 31/63] feat: add sort/group logic for json --- source/libs/scalar/src/sclvector.c | 5 ++--- source/libs/scalar/test/scalar/scalarTests.cpp | 4 ++-- tests/system-test/2-query/json_tag.py | 12 ++++++------ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index b931badab9..9b3a8261cc 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -393,7 +393,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in pOut->numOfRows = pIn->numOfRows; for (int32_t i = 0; i < pIn->numOfRows; ++i) { - if (colDataIsNull_s(pIn->columnData, i)) { + if (IS_HELPER_NULL(pIn->columnData, i)) { colDataAppendNULL(pOut->columnData, i); continue; } @@ -402,8 +402,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in int32_t convertType = inType; if(inType == TSDB_DATA_TYPE_JSON){ if(*data == TSDB_DATA_TYPE_NULL) { - colDataAppendNULL(pOut->columnData, i); - continue; + ASSERT(0); } else if(*data == TSDB_DATA_TYPE_NCHAR) { data += CHAR_BYTES; diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index c71521535a..8a29462a2b 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -1227,7 +1227,7 @@ TEST(columnTest, json_column_logic_op) { printf("--------------------json null---------------------\n"); key = "k3"; // (null is true) return NULL, so use DBL_MAX represent NULL - double eRes2[len+len1] = {false, false, false, false, false, false, true, false, false, false, false, false, false}; + double eRes2[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]); } @@ -1279,7 +1279,7 @@ TEST(columnTest, json_column_logic_op) { printf("---------------------json not exist--------------------\n"); key = "k10"; // (NULL is true) return NULL, so use DBL_MAX represent NULL - double eRes10[len+len1] = {false, false, false, false, false, false, true, false, false, false, false, false, false}; + double eRes10[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i]); } diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 33f35eabbe..5c38d4afb9 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -389,8 +389,8 @@ class TDTestCase: tdSql.checkData(2, 1, "11.000000000") tdSql.checkData(5, 0, 1) tdSql.checkData(5, 1, "false") - # tdSql.checkData(6, 0, 1) - # tdSql.checkData(6, 1, "null") + tdSql.checkData(6, 0, 1) + tdSql.checkData(6, 1, "null") tdSql.checkData(7, 0, 2) tdSql.checkData(7, 1, None) @@ -409,7 +409,7 @@ class TDTestCase: tdSql.query("select stddev(dataint),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") tdSql.checkRows(8) tdSql.checkData(0, 0, 10) - # tdSql.checkData(0, 1, None) + tdSql.checkData(0, 1, None) tdSql.checkData(4, 0, 0) tdSql.checkData(4, 1, "5.000000000") tdSql.checkData(7, 0, 11) @@ -424,10 +424,10 @@ class TDTestCase: # test top/bottom with group by json tag tdSql.query("select top(dataint,2),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") tdSql.checkRows(11) + tdSql.checkData(0, 0, 24) + tdSql.checkData(0, 1, None) tdSql.checkData(3, 0, 3) tdSql.checkData(3, 1, "false") - # tdSql.checkData(3, 0, 24) - # tdSql.checkData(3, 1, None) tdSql.checkData(10, 0, 23) tdSql.checkData(10, 1, '"femail"') @@ -436,7 +436,7 @@ class TDTestCase: # tdSql.checkRows(2) # subquery with json tag - tdSql.query("select * from (select jtag, dataint from jsons1)") + tdSql.query("select * from (select jtag, dataint from jsons1) order by dataint") tdSql.checkRows(11) tdSql.checkData(1, 1, 1) tdSql.checkData(2, 0, '{"tag1":5,"tag2":"beijing"}') From f425ccf23b1178807b135fa22ac38d8f7fc7c403 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 5 Jun 2022 14:38:10 +0800 Subject: [PATCH 32/63] handle conflict --- source/libs/index/src/indexFilter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 2d8522a49b..cb605ca073 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -508,7 +508,7 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { int32_t code = 0; int32_t nParam = sifGetOperParamNum(node->opType); if (nParam <= 1) { - SIF_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } if (node->opType == OP_TYPE_JSON_GET_VALUE || node->opType == OP_TYPE_JSON_CONTAINS) { return code; From 6fcbcabdfc14ed3701332ac3b1406d9f3ef576c7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 5 Jun 2022 14:48:15 +0800 Subject: [PATCH 33/63] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 13 ++- source/libs/executor/src/executil.c | 41 -------- source/libs/executor/src/executorimpl.c | 127 ++++++------------------ source/libs/executor/src/scanoperator.c | 92 +++++++++-------- source/libs/executor/src/sortoperator.c | 15 +++ 5 files changed, 105 insertions(+), 183 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 936e9307d1..061bdc7236 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -727,7 +727,11 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR char* pData, int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup); +SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo); + SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo); @@ -738,9 +742,6 @@ SOperatorInfo *createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName, - SNode* pCondition, SEpSet epset, SArray* colList, - SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); @@ -758,9 +759,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo); SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle, - SArray* pTableIdList, STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, - STimeWindowAggSupp* pTwSup); +SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle, SArray* pTableIdList, + STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup); SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, @@ -776,7 +776,6 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput, SSDataBlock* pResBlock, SArray* pColMatchInfo, STableListInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 1c45e38b63..484ff35e3a 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -28,18 +28,6 @@ typedef struct SCompSupporter { int32_t order; } SCompSupporter; -int32_t getRowNumForMultioutput(STaskAttr* pQueryAttr, bool topBottomQuery, bool stable) { - if (pQueryAttr && (!stable)) { - for (int16_t i = 0; i < pQueryAttr->numOfOutput; ++i) { -// if (pQueryAttr->pExpr1[i].base. == FUNCTION_TOP || pQueryAttr->pExpr1[i].base.functionId == FUNCTION_BOTTOM) { -// return (int32_t)pQueryAttr->pExpr1[i].base.param[0].i; -// } - } - } - - return 1; -} - int32_t getOutputInterResultBufSize(STaskAttr* pQueryAttr) { int32_t size = 0; @@ -113,35 +101,6 @@ void closeResultRow(SResultRow* pResultRow) { pResultRow->closed = true; } -void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow) { - if (pResultRow == NULL) { - return; - } - - // the result does not put into the SDiskbasedBuf, ignore it. - if (pResultRow->pageId >= 0) { - SFilePage *page = getBufPage(pRuntimeEnv->pResultBuf, pResultRow->pageId); - - int16_t offset = 0; - for (int32_t i = 0; i < pRuntimeEnv->pQueryAttr->numOfOutput; ++i) { - struct SResultRowEntryInfo *pEntryInfo = NULL;//pResultRow->pEntryInfo[i]; - -// int16_t size = pRuntimeEnv->pQueryAttr->pExpr1[i].base.resSchema.bytes; -// char * s = getPosInResultPage(pRuntimeEnv->pQueryAttr, page, pResultRow->offset, offset); -// memset(s, 0, size); - -// offset += size; - cleanupResultRowEntry(pEntryInfo); - } - } - - pResultRow->numOfRows = 0; - pResultRow->pageId = -1; - pResultRow->offset = -1; - pResultRow->closed = false; - pResultRow->win = TSWINDOW_INITIALIZER; -} - // TODO refactor: use macro SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, const int32_t* offset) { assert(index >= 0 && offset != NULL); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 1a73e9f3ac..6434e3a016 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2896,52 +2896,55 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo) { return TSDB_CODE_SUCCESS; } -SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, const SNodeList* pSources, SSDataBlock* pBlock, - SExecTaskInfo* pTaskInfo) { +static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* pInfo) { + size_t numOfSources = LIST_LENGTH(pExNode->pSrcEndPoints); + + pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode)); + pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); + if (pInfo->pSourceDataInfo == NULL || pInfo->pSources == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < numOfSources; ++i) { + SNodeListNode* pNode = nodesListGetNode((SNodeList*)pExNode->pSrcEndPoints, i); + taosArrayPush(pInfo->pSources, pNode); + } + + return initDataSource(numOfSources, pInfo); +} + +SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo) { SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - size_t numOfSources = LIST_LENGTH(pSources); - pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode)); - pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); - if (pInfo->pSourceDataInfo == NULL || pInfo->pSources == NULL) { - goto _error; - } - - for (int32_t i = 0; i < numOfSources; ++i) { - SNodeListNode* pNode = nodesListGetNode((SNodeList*)pSources, i); - taosArrayPush(pInfo->pSources, pNode); - } - - int32_t code = initDataSource(numOfSources, pInfo); + int32_t code = initExchangeOperator(pExNode, pInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; } - pInfo->pResult = pBlock; - pInfo->seqLoadData = false; - + pInfo->seqLoadData = false; + pInfo->pTransporter = pTransporter; + pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); tsem_init(&pInfo->ready, 0, 0); - pOperator->name = "ExchangeOperator"; + pOperator->name = "ExchangeOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfExprs = pBlock->info.numOfCols; - pOperator->pTaskInfo = pTaskInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfExprs = pInfo->pResult->info.numOfCols; + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, destroyExchangeOperatorInfo, NULL, NULL, NULL); - pInfo->pTransporter = pTransporter; return pOperator; _error: if (pInfo != NULL) { - destroyExchangeOperatorInfo(pInfo, numOfSources); + destroyExchangeOperatorInfo(pInfo, LIST_LENGTH(pExNode->pSrcEndPoints)); } taosMemoryFreeClear(pInfo); @@ -4394,9 +4397,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { - SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; - SSDataBlock* pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc); - return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, pExchange->pSrcEndPoints, pResBlock, pTaskInfo); + return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; @@ -4424,41 +4425,16 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; - SScanPhysiNode* pScanNode = &pSysScanPhyNode->scan; - - SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; - - SSDataBlock* pResBlock = createResDataBlock(pDescNode); - - int32_t numOfOutputCols = 0; - SArray* colList = - extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_COL_ID); - SOperatorInfo* pOperator = createSysTableScanOperatorInfo( - pHandle, pResBlock, &pScanNode->tableName, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, colList, - pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId); - return pOperator; + return createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; - SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; - - SSDataBlock* pResBlock = createResDataBlock(pDescNode); - int32_t code = getTableList(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableListInfo, pTagCond); if (code != TSDB_CODE_SUCCESS) { return NULL; } - int32_t num = 0; - SExprInfo* pExprInfo = createExprInfo(pScanPhyNode->pScanPseudoCols, NULL, &num); - - int32_t numOfOutputCols = 0; - SArray* colList = extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pDescNode, &numOfOutputCols, pTaskInfo, - COL_MATCH_FROM_COL_ID); - - SOperatorInfo* pOperator = - createTagScanOperatorInfo(pHandle, pExprInfo, num, pResBlock, colList, pTableListInfo, pTaskInfo); - return pOperator; + return createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo); } else { ASSERT(0); } @@ -5016,45 +4992,6 @@ _complete: return code; } -void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo) { - const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512); - - // the minimum number of rows for projection query - const int32_t MIN_ROWS_FOR_PRJ_QUERY = 8192; - const int32_t DEFAULT_MIN_ROWS = 4096; - - const float THRESHOLD_RATIO = 0.85f; - - // if (isProjQuery(pQueryAttr)) { - // int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQueryAttr->resultRowSize; - // if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) { - // numOfRes = MIN_ROWS_FOR_PRJ_QUERY; - // } - // - // pResultInfo->capacity = numOfRes; - // } else { // in case of non-prj query, a smaller output buffer will be used. - // pResultInfo->capacity = DEFAULT_MIN_ROWS; - // } - - pResultInfo->threshold = (int32_t)(pResultInfo->capacity * THRESHOLD_RATIO); - pResultInfo->totalRows = 0; -} - -// TODO refactor -void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters) { - if (pFilter == NULL || numOfFilters == 0) { - return; - } - - for (int32_t i = 0; i < numOfFilters; i++) { - if (pFilter[i].filterstr && pFilter[i].pz) { - taosMemoryFree((void*)(pFilter[i].pz)); - } - } - - taosMemoryFree(pFilter); -} - static void doDestroyTableList(STableListInfo* pTableqinfoList) { taosArrayDestroy(pTableqinfoList->pTableList); taosHashCleanup(pTableqinfoList->map); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1633d2f4e6..8541a0b129 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -36,11 +36,6 @@ #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) -typedef struct SWindowPosition { - int32_t pageId; - int32_t rowId; -} SWindowPosition; - static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity); static int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, const char* dbName); @@ -1430,48 +1425,59 @@ int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbT return numOfRows; } -SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pResBlock, const SName* pName, - SNode* pCondition, SEpSet epset, SArray* colList, - SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId) { +SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, SExecTaskInfo* pTaskInfo) { SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; + goto _error; } - pInfo->accountId = accountId; - pInfo->showRewrite = showRewrite; - pInfo->pRes = pResBlock; - pInfo->pCondition = pCondition; - pInfo->scanCols = colList; + SScanPhysiNode* pScanNode = &pScanPhyNode->scan; + + SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; + SSDataBlock* pResBlock = createResDataBlock(pDescNode); + + int32_t num = 0; + SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, pTaskInfo, COL_MATCH_FROM_COL_ID); + + pInfo->accountId = pScanPhyNode->accountId; + pInfo->showRewrite = pScanPhyNode->showRewrite; + pInfo->pRes = pResBlock; + pInfo->pCondition = pScanNode->node.pConditions; + pInfo->scanCols = colList; initResultSizeInfo(pOperator, 4096); - tNameAssign(&pInfo->name, pName); + tNameAssign(&pInfo->name, &pScanNode->tableName); const char* name = tNameGetTableName(&pInfo->name); + if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { pInfo->readHandle = *(SReadHandle*)readHandle; blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); } else { tsem_init(&pInfo->ready, 0, 0); - pInfo->epSet = epset; + pInfo->epSet = pScanPhyNode->mgmtEpSet; pInfo->readHandle = *(SReadHandle*)readHandle; } - pOperator->name = "SysTableScanOperator"; + pOperator->name = "SysTableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfExprs = pResBlock->info.numOfCols; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfExprs = pResBlock->info.numOfCols; + pOperator->pTaskInfo = pTaskInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, NULL, destroySysScanOperator, NULL, NULL, NULL); - pOperator->pTaskInfo = pTaskInfo; return pOperator; + + _error: + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; } static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { @@ -1626,28 +1632,34 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } -SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput, - SSDataBlock* pResBlock, SArray* pColMatchInfo, STableListInfo* pTableListInfo, - SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->pTableList = pTableListInfo; - pInfo->pColMatchInfo = pColMatchInfo; - pInfo->pRes = pResBlock; - pInfo->readHandle = *pReadHandle; - pInfo->curPos = 0; - pOperator->name = "TagScanOperator"; + SDataBlockDescNode* pDescNode = pPhyNode->node.pOutputDataBlockDesc; + + int32_t numOfExprs = 0; + SExprInfo* pExprInfo = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &numOfExprs); + + int32_t num = 0; + SArray* colList = extractColMatchInfo(pPhyNode->pScanPseudoCols, pDescNode, &num, pTaskInfo, COL_MATCH_FROM_COL_ID); + + pInfo->pTableList = pTableListInfo; + pInfo->pColMatchInfo = colList; + pInfo->pRes = createResDataBlock(pDescNode);; + pInfo->readHandle = *pReadHandle; + pInfo->curPos = 0; + pOperator->name = "TagScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfExprs = numOfOutput; - pOperator->pTaskInfo = pTaskInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pExpr = pExprInfo; + pOperator->numOfExprs = numOfExprs; + pOperator->pTaskInfo = pTaskInfo; initResultSizeInfo(pOperator, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 8f5fa88070..5e1b47cc91 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #include "tdatablock.h" #include "executorimpl.h" From 980e289cdfafd8630e526bec016be46019f436d4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 5 Jun 2022 14:59:53 +0800 Subject: [PATCH 34/63] refactor: add alter-confirm while alter db --- include/common/tmsgdef.h | 6 +- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 1 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/mnode/impl/inc/mndVgroup.h | 2 +- source/dnode/mnode/impl/src/mndBnode.c | 32 +++---- source/dnode/mnode/impl/src/mndDb.c | 96 ++++++++++++++------- source/dnode/mnode/impl/src/mndQnode.c | 32 +++---- source/dnode/mnode/impl/src/mndSma.c | 32 +++---- source/dnode/mnode/impl/src/mndSnode.c | 32 +++---- source/dnode/mnode/impl/src/mndStb.c | 24 +----- source/dnode/mnode/impl/src/mndStream.c | 26 +++--- source/dnode/mnode/impl/src/mndSubscribe.c | 28 +++--- source/dnode/mnode/impl/src/mndTopic.c | 24 +++--- source/dnode/mnode/impl/src/mndVgroup.c | 86 +++++++----------- source/dnode/vnode/src/vnd/vnodeSvr.c | 81 ++++++++++------- 15 files changed, 232 insertions(+), 271 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 012bc40c8a..73f8515f22 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -193,13 +193,11 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp) TD_DEF_MSG_TYPE(TDMT_VND_GET_TSMA_EXP_WNDS, "vnode-get-tsma-expired-windows", SVGetTsmaExpWndsReq, SVGetTsmaExpWndsRsp) + TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_START_WRITE, "start-write", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STOP_WRITE, "stop-write", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_CONFIRM_WRITE, "confirm-write", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "compact", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp) TD_NEW_MSG_SEG(TDMT_QND_MSG) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 78ab850964..27f5e6f7af 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -213,6 +213,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_DEPLOY_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 0819f79cf9..77109ab52f 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -361,6 +361,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 3f4f3f2053..2119320de5 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -34,7 +34,7 @@ int32_t mndAllocSmaVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup); int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); SArray *mndBuildDnodesArray(SMnode *pMnode); int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray); -int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2); +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pVgId); void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen, bool standby); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 801f335a80..38a166900d 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -30,25 +30,25 @@ static int32_t mndBnodeActionInsert(SSdb *pSdb, SBnodeObj *pObj); static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOld, SBnodeObj *pNew); static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj); static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq); -static int32_t mndProcessCreateBnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq); -static int32_t mndProcessDropBnodeRsp(SRpcMsg *pRsp); static int32_t mndRetrieveBnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter); int32_t mndInitBnode(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_BNODE, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndBnodeActionEncode, - .decodeFp = (SdbDecodeFp)mndBnodeActionDecode, - .insertFp = (SdbInsertFp)mndBnodeActionInsert, - .updateFp = (SdbUpdateFp)mndBnodeActionUpdate, - .deleteFp = (SdbDeleteFp)mndBnodeActionDelete}; + SSdbTable table = { + .sdbType = SDB_BNODE, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndBnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndBnodeActionDecode, + .insertFp = (SdbInsertFp)mndBnodeActionInsert, + .updateFp = (SdbUpdateFp)mndBnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndBnodeActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_BNODE, mndProcessCreateBnodeReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_BNODE, mndProcessDropBnodeReq); - mndSetMsgHandle(pMnode, TDMT_DND_CREATE_BNODE_RSP, mndProcessCreateBnodeRsp); - mndSetMsgHandle(pMnode, TDMT_DND_DROP_BNODE_RSP, mndProcessDropBnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_BNODE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_BNODE_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_BNODE, mndRetrieveBnodes); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_BNODE, mndCancelGetNextBnode); @@ -427,16 +427,6 @@ _OVER: return code; } -static int32_t mndProcessCreateBnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - -static int32_t mndProcessDropBnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndRetrieveBnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index bbd80d765a..1ce1a2590a 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -263,7 +263,8 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } -static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool standby) { +static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool standby) { STransAction action = {0}; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); @@ -288,6 +289,32 @@ static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } +static int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + + int32_t contLen = sizeof(SMsgHead); + SMsgHead *pHead = taosMemoryMalloc(contLen); + if (pHead == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + action.pCont = pHead; + action.contLen = contLen; + action.msgType = TDMT_VND_ALTER_CONFIRM; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pHead); + return -1; + } + + return 0; +} + static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType) { STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); @@ -415,7 +442,6 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0; if (pCfg->schemaless < 0) pCfg->schemaless = TSDB_DB_SCHEMALESS_OFF; - } static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { @@ -726,30 +752,32 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj mndTransSetSerial(pTrans); if (newVgroup.replica < pDb->cfg.replications) { - mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId, + mInfo("db:%s, vgId:%d, vn:0 dnode:%d, will add 2 vnodes", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); - if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) { - mError("db:%s, failed to add vnode to vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); - return -1; - } - newVgroup.replica = pDb->cfg.replications; + if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; + + if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1; if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; } else { mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId); SVnodeGid del1 = {0}; - SVnodeGid del2 = {0}; - if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1, &del2) != 0) { - mError("db:%s, failed to remove vnode from vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); - return -1; - } - newVgroup.replica = pDb->cfg.replications; + if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1; if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; + + SVnodeGid del2 = {0}; + if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; } SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); @@ -1341,7 +1369,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable) { - mDebug("db:%s, version & numOfTable not changed", pDbVgVersion->dbFName); + mDebug("db:%s, version and numOfTable not changed", pDbVgVersion->dbFName); mndReleaseDb(pMnode, pDb); continue; } else { @@ -1433,12 +1461,22 @@ const char *mndGetDbStr(const char *src) { int64_t getValOfDiffPrecision(int8_t unit, int64_t val) { int64_t v = 0; - switch(unit) { - case 's': v = val / 1000; break; - case 'm': v = val / tsTickPerMin[TSDB_TIME_PRECISION_MILLI]; break; - case 'h': v = val / (tsTickPerMin[TSDB_TIME_PRECISION_MILLI] * 60); break; - case 'd': v = val / (tsTickPerMin[TSDB_TIME_PRECISION_MILLI] * 24 * 60); break; - case 'w': v = val / (tsTickPerMin[TSDB_TIME_PRECISION_MILLI] * 24 * 60 * 7); break; + switch (unit) { + case 's': + v = val / 1000; + break; + case 'm': + v = val / tsTickPerMin[TSDB_TIME_PRECISION_MILLI]; + break; + case 'h': + v = val / (tsTickPerMin[TSDB_TIME_PRECISION_MILLI] * 60); + break; + case 'd': + v = val / (tsTickPerMin[TSDB_TIME_PRECISION_MILLI] * 24 * 60); + break; + case 'w': + v = val / (tsTickPerMin[TSDB_TIME_PRECISION_MILLI] * 24 * 60 * 7); + break; default: break; } @@ -1446,32 +1484,32 @@ int64_t getValOfDiffPrecision(int8_t unit, int64_t val) { return v; } -char* buildRetension(SArray* pRetension) { +char *buildRetension(SArray *pRetension) { size_t size = taosArrayGetSize(pRetension); if (size == 0) { return NULL; } - char* p1 = taosMemoryCalloc(1, 100); - SRetention* p = taosArrayGet(pRetension, 0); + char *p1 = taosMemoryCalloc(1, 100); + SRetention *p = taosArrayGet(pRetension, 0); int32_t len = 2; int64_t v1 = getValOfDiffPrecision(p->freqUnit, p->freq); int64_t v2 = getValOfDiffPrecision(p->keepUnit, p->keep); - len += sprintf(p1 + len, "%"PRId64"%c:%"PRId64"%c,", v1, p->freqUnit, v2, p->keepUnit); + len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c,", v1, p->freqUnit, v2, p->keepUnit); p = taosArrayGet(pRetension, 1); v1 = getValOfDiffPrecision(p->freqUnit, p->freq); v2 = getValOfDiffPrecision(p->keepUnit, p->keep); - len += sprintf(p1 + len, "%"PRId64"%c:%"PRId64"%c,", v1, p->freqUnit, v2, p->keepUnit); + len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c,", v1, p->freqUnit, v2, p->keepUnit); p = taosArrayGet(pRetension, 2); v1 = getValOfDiffPrecision(p->freqUnit, p->freq); v2 = getValOfDiffPrecision(p->keepUnit, p->keep); - len += sprintf(p1 + len, "%"PRId64"%c:%"PRId64"%c", v1, p->freqUnit, v2, p->keepUnit); + len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit); varDataSetLen(p1, len); return p1; @@ -1586,7 +1624,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in } colDataAppend(pColInfo, rows, cacheModel, null); #endif - colDataAppend(pColInfo, rows, (const char*) &pDb->cfg.cacheLastRow, false); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLastRow, false); char *prec = NULL; switch (pDb->cfg.precision) { @@ -1618,7 +1656,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); - char* p = buildRetension(pDb->cfg.pRetensions); + char *p = buildRetension(pDb->cfg.pRetensions); pColInfo = taosArrayGet(pBlock->pDataBlock, cols); if (p == NULL) { diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 27881865af..4fb9226144 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -30,26 +30,26 @@ static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj); static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOld, SQnodeObj *pNew); static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj); static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq); -static int32_t mndProcessCreateQnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq); -static int32_t mndProcessDropQnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessQnodeListReq(SRpcMsg *pReq); static int32_t mndRetrieveQnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextQnode(SMnode *pMnode, void *pIter); int32_t mndInitQnode(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_QNODE, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndQnodeActionEncode, - .decodeFp = (SdbDecodeFp)mndQnodeActionDecode, - .insertFp = (SdbInsertFp)mndQnodeActionInsert, - .updateFp = (SdbUpdateFp)mndQnodeActionUpdate, - .deleteFp = (SdbDeleteFp)mndQnodeActionDelete}; + SSdbTable table = { + .sdbType = SDB_QNODE, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndQnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndQnodeActionDecode, + .insertFp = (SdbInsertFp)mndQnodeActionInsert, + .updateFp = (SdbUpdateFp)mndQnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndQnodeActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_QNODE, mndProcessCreateQnodeReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_QNODE, mndProcessDropQnodeReq); - mndSetMsgHandle(pMnode, TDMT_DND_CREATE_QNODE_RSP, mndProcessCreateQnodeRsp); - mndSetMsgHandle(pMnode, TDMT_DND_DROP_QNODE_RSP, mndProcessDropQnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_QNODE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_QNODE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_QNODE_LIST, mndProcessQnodeListReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QNODE, mndRetrieveQnodes); @@ -503,16 +503,6 @@ _OVER: return code; } -static int32_t mndProcessCreateQnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - -static int32_t mndProcessDropQnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndRetrieveQnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 6cb70d1f27..e8e23ccc2a 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -38,25 +38,25 @@ static int32_t mndSmaActionDelete(SSdb *pSdb, SSmaObj *pSpSmatb); static int32_t mndSmaActionUpdate(SSdb *pSdb, SSmaObj *pOld, SSmaObj *pNew); static int32_t mndProcessMCreateSmaReq(SRpcMsg *pReq); static int32_t mndProcessMDropSmaReq(SRpcMsg *pReq); -static int32_t mndProcessVCreateSmaRsp(SRpcMsg *pRsp); -static int32_t mndProcessVDropSmaRsp(SRpcMsg *pRsp); static int32_t mndProcessGetSmaReq(SRpcMsg *pReq); static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSma(SMnode *pMnode, void *pIter); int32_t mndInitSma(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_SMA, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndSmaActionEncode, - .decodeFp = (SdbDecodeFp)mndSmaActionDecode, - .insertFp = (SdbInsertFp)mndSmaActionInsert, - .updateFp = (SdbUpdateFp)mndSmaActionUpdate, - .deleteFp = (SdbDeleteFp)mndSmaActionDelete}; + SSdbTable table = { + .sdbType = SDB_SMA, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndSmaActionEncode, + .decodeFp = (SdbDecodeFp)mndSmaActionDecode, + .insertFp = (SdbInsertFp)mndSmaActionInsert, + .updateFp = (SdbUpdateFp)mndSmaActionUpdate, + .deleteFp = (SdbDeleteFp)mndSmaActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SMA, mndProcessMCreateSmaReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessMDropSmaReq); - mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndProcessVCreateSmaRsp); - mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndProcessVDropSmaRsp); + mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetSmaReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndRetrieveSma); @@ -637,11 +637,6 @@ _OVER: return code; } -static int32_t mndProcessVCreateSmaRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndSetDropSmaRedoLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { SSdbRaw *pRedoRaw = mndSmaActionEncode(pSma); if (pRedoRaw == NULL) return -1; @@ -910,11 +905,6 @@ _OVER: return code; } -static int32_t mndProcessVDropSmaRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index c6acb4fef4..f5a1e76723 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -30,25 +30,25 @@ static int32_t mndSnodeActionInsert(SSdb *pSdb, SSnodeObj *pObj); static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOld, SSnodeObj *pNew); static int32_t mndSnodeActionDelete(SSdb *pSdb, SSnodeObj *pObj); static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq); -static int32_t mndProcessCreateSnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq); -static int32_t mndProcessDropSnodeRsp(SRpcMsg *pRsp); static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSnode(SMnode *pMnode, void *pIter); int32_t mndInitSnode(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_SNODE, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndSnodeActionEncode, - .decodeFp = (SdbDecodeFp)mndSnodeActionDecode, - .insertFp = (SdbInsertFp)mndSnodeActionInsert, - .updateFp = (SdbUpdateFp)mndSnodeActionUpdate, - .deleteFp = (SdbDeleteFp)mndSnodeActionDelete}; + SSdbTable table = { + .sdbType = SDB_SNODE, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndSnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndSnodeActionDecode, + .insertFp = (SdbInsertFp)mndSnodeActionInsert, + .updateFp = (SdbUpdateFp)mndSnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndSnodeActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SNODE, mndProcessCreateSnodeReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_SNODE, mndProcessDropSnodeReq); - mndSetMsgHandle(pMnode, TDMT_DND_CREATE_SNODE_RSP, mndProcessCreateSnodeRsp); - mndSetMsgHandle(pMnode, TDMT_DND_DROP_SNODE_RSP, mndProcessDropSnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_SNODE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_SNODE_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_SNODE, mndRetrieveSnodes); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_SNODE, mndCancelGetNextSnode); @@ -437,16 +437,6 @@ _OVER: return code; } -static int32_t mndProcessCreateSnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - -static int32_t mndProcessDropSnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 556837f397..175878959d 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -38,9 +38,6 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew); static int32_t mndProcessMCreateStbReq(SRpcMsg *pReq); static int32_t mndProcessMAlterStbReq(SRpcMsg *pReq); static int32_t mndProcessMDropStbReq(SRpcMsg *pReq); -static int32_t mndProcessVCreateStbRsp(SRpcMsg *pRsp); -static int32_t mndProcessVAlterStbRsp(SRpcMsg *pRsp); -static int32_t mndProcessVDropStbRsp(SRpcMsg *pRsp); static int32_t mndProcessTableMetaReq(SRpcMsg *pReq); static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); @@ -59,9 +56,9 @@ int32_t mndInitStb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STB, mndProcessMCreateStbReq); mndSetMsgHandle(pMnode, TDMT_MND_ALTER_STB, mndProcessMAlterStbReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_STB, mndProcessMDropStbReq); - mndSetMsgHandle(pMnode, TDMT_VND_CREATE_STB_RSP, mndProcessVCreateStbRsp); - mndSetMsgHandle(pMnode, TDMT_VND_ALTER_STB_RSP, mndProcessVAlterStbRsp); - mndSetMsgHandle(pMnode, TDMT_VND_DROP_STB_RSP, mndProcessVDropStbRsp); + mndSetMsgHandle(pMnode, TDMT_VND_CREATE_STB_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_ALTER_STB_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_STB_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STB, mndRetrieveStb); @@ -837,11 +834,6 @@ _OVER: return code; } -static int32_t mndProcessVCreateStbRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) { if (pAlter->commentLen != 0 || pAlter->ttl != 0) return 0; @@ -1459,11 +1451,6 @@ _OVER: return code; } -static int32_t mndProcessVAlterStbRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndSetDropStbRedoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); if (pRedoRaw == NULL) return -1; @@ -1599,11 +1586,6 @@ _OVER: return code; } -static int32_t mndProcessVDropStbRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5ee5b06a57..a0aa2067d3 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -35,7 +35,6 @@ static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq); -static int32_t mndProcessTaskDeployInternalRsp(SRpcMsg *pRsp); /*static int32_t mndProcessDropStreamReq(SRpcMsg *pReq);*/ /*static int32_t mndProcessDropStreamInRsp(SRpcMsg *pRsp);*/ static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq); @@ -44,17 +43,19 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB static void mndCancelGetNextStream(SMnode *pMnode, void *pIter); int32_t mndInitStream(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_STREAM, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndStreamActionEncode, - .decodeFp = (SdbDecodeFp)mndStreamActionDecode, - .insertFp = (SdbInsertFp)mndStreamActionInsert, - .updateFp = (SdbUpdateFp)mndStreamActionUpdate, - .deleteFp = (SdbDeleteFp)mndStreamActionDelete}; + SSdbTable table = { + .sdbType = SDB_STREAM, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndStreamActionEncode, + .decodeFp = (SdbDecodeFp)mndStreamActionDecode, + .insertFp = (SdbInsertFp)mndStreamActionInsert, + .updateFp = (SdbUpdateFp)mndStreamActionUpdate, + .deleteFp = (SdbDeleteFp)mndStreamActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq); - mndSetMsgHandle(pMnode, TDMT_VND_TASK_DEPLOY_RSP, mndProcessTaskDeployInternalRsp); - mndSetMsgHandle(pMnode, TDMT_SND_TASK_DEPLOY_RSP, mndProcessTaskDeployInternalRsp); + mndSetMsgHandle(pMnode, TDMT_VND_TASK_DEPLOY_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_SND_TASK_DEPLOY_RSP, mndTransProcessRsp); /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq);*/ /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/ @@ -195,11 +196,6 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) { sdbRelease(pSdb, pStream); } -static int32_t mndProcessTaskDeployInternalRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static SDbObj *mndAcquireDbByStream(SMnode *pMnode, char *streamName) { SName name = {0}; tNameFromString(&name, streamName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index c7f8415b65..cbaf8bc1d1 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -43,7 +43,6 @@ static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubs static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg); static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg); -static int32_t mndProcessSubscribeInternalRsp(SRpcMsg *pMsg); static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter); @@ -65,20 +64,22 @@ static int32_t mndSetSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeO } int32_t mndInitSubscribe(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_SUBSCRIBE, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndSubActionEncode, - .decodeFp = (SdbDecodeFp)mndSubActionDecode, - .insertFp = (SdbInsertFp)mndSubActionInsert, - .updateFp = (SdbUpdateFp)mndSubActionUpdate, - .deleteFp = (SdbDeleteFp)mndSubActionDelete}; + SSdbTable table = { + .sdbType = SDB_SUBSCRIBE, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndSubActionEncode, + .decodeFp = (SdbDecodeFp)mndSubActionDecode, + .insertFp = (SdbInsertFp)mndSubActionInsert, + .updateFp = (SdbUpdateFp)mndSubActionUpdate, + .deleteFp = (SdbDeleteFp)mndSubActionDelete, + }; - mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_CHANGE_RSP, mndProcessSubscribeInternalRsp); - mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_DELETE_RSP, mndProcessSubscribeInternalRsp); + mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_CHANGE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_DELETE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq); mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq); mndSetMsgHandle(pMnode, TDMT_MND_MQ_DROP_CGROUP, mndProcessDropCgroupReq); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_DROP_CGROUP_RSP, mndProcessSubscribeInternalRsp); + mndSetMsgHandle(pMnode, TDMT_MND_MQ_DROP_CGROUP_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_SUBSCRIPTIONS, mndRetrieveSubscribe); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextSubscribe); @@ -789,11 +790,6 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) { sdbRelease(pSdb, pSub); } -static int32_t mndProcessSubscribeInternalRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndSetDropSubRedoLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) { SSdbRaw *pRedoRaw = mndSubActionEncode(pSub); if (pRedoRaw == NULL) return -1; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index e0d565c9af..95f22a33db 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -37,7 +37,6 @@ static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic); static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pTopic, SMqTopicObj *pNewTopic); static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq); static int32_t mndProcessDropTopicReq(SRpcMsg *pReq); -static int32_t mndProcessDropTopicInRsp(SRpcMsg *pRsp); static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); @@ -45,17 +44,19 @@ static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); static int32_t mndSetDropTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic); int32_t mndInitTopic(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_TOPIC, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndTopicActionEncode, - .decodeFp = (SdbDecodeFp)mndTopicActionDecode, - .insertFp = (SdbInsertFp)mndTopicActionInsert, - .updateFp = (SdbUpdateFp)mndTopicActionUpdate, - .deleteFp = (SdbDeleteFp)mndTopicActionDelete}; + SSdbTable table = { + .sdbType = SDB_TOPIC, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndTopicActionEncode, + .decodeFp = (SdbDecodeFp)mndTopicActionDecode, + .insertFp = (SdbInsertFp)mndTopicActionInsert, + .updateFp = (SdbUpdateFp)mndTopicActionUpdate, + .deleteFp = (SdbDeleteFp)mndTopicActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_TOPIC, mndProcessCreateTopicReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_TOPIC, mndProcessDropTopicReq); - mndSetMsgHandle(pMnode, TDMT_VND_DROP_TOPIC_RSP, mndProcessDropTopicInRsp); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_TOPIC_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveTopic); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextTopic); @@ -607,11 +608,6 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { return TSDB_CODE_ACTION_IN_PROGRESS; } -static int32_t mndProcessDropTopicInRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 2577febf66..4f09eda733 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -29,11 +29,6 @@ static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew); -static int32_t mndProcessCreateVnodeRsp(SRpcMsg *pRsp); -static int32_t mndProcessAlterVnodeRsp(SRpcMsg *pRsp); -static int32_t mndProcessDropVnodeRsp(SRpcMsg *pRsp); -static int32_t mndProcessCompactVnodeRsp(SRpcMsg *pRsp); - static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); @@ -50,11 +45,12 @@ int32_t mndInitVgroup(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndVgroupActionDelete, }; - mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp); - mndSetMsgHandle(pMnode, TDMT_VND_ALTER_REPLICA_RSP, mndProcessAlterVnodeRsp); - mndSetMsgHandle(pMnode, TDMT_VND_ALTER_CONFIG_RSP, mndProcessAlterVnodeRsp); - mndSetMsgHandle(pMnode, TDMT_DND_DROP_VNODE_RSP, mndProcessDropVnodeRsp); - mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_RSP, mndProcessCompactVnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_ALTER_REPLICA_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_ALTER_CONFIG_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_ALTER_CONFIRM_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_VNODE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndRetrieveVgroups); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndCancelGetNextVgroup); @@ -512,12 +508,12 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); } - int32_t maxPos = 1; + SVnodeGid *pVgid = &pVgroup->vnodeGid[pVgroup->replica]; for (int32_t d = 0; d < taosArrayGetSize(pArray); ++d) { SDnodeObj *pDnode = taosArrayGet(pArray, d); bool used = false; - for (int32_t vn = 0; vn < maxPos; ++vn) { + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { if (pDnode->id == pVgroup->vnodeGid[vn].dnodeId) { used = true; break; @@ -530,59 +526,58 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { return -1; } - SVnodeGid *pVgid = &pVgroup->vnodeGid[maxPos]; pVgid->dnodeId = pDnode->id; pVgid->role = TAOS_SYNC_STATE_ERROR; - pDnode->numOfVnodes++; + mInfo("db:%s, vgId:%d, vn:%d dnode:%d, is added", pVgroup->dbName, pVgroup->vgId, pVgroup->replica, + pVgid->dnodeId); - mInfo("db:%s, vgId:%d, vnode_index:%d dnode:%d is added", pVgroup->dbName, pVgroup->vgId, maxPos, pVgid->dnodeId); - maxPos++; - if (maxPos == 3) return 0; + pVgroup->replica++; + pDnode->numOfVnodes++; + return 0; } terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + mError("db:%s, failed to add vnode to vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr()); return -1; } -int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2) { +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pDelVgid) { taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { SDnodeObj *pDnode = taosArrayGet(pArray, i); mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); } - int32_t removedNum = 0; + int32_t code = -1; for (int32_t d = taosArrayGetSize(pArray) - 1; d >= 0; --d) { SDnodeObj *pDnode = taosArrayGet(pArray, d); - for (int32_t vn = 0; vn < TSDB_MAX_REPLICA; ++vn) { + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; if (pVgid->dnodeId == pDnode->id) { - if (removedNum == 0) *del1 = *pVgid; - if (removedNum == 1) *del2 = *pVgid; - - mInfo("db:%s, vgId:%d, vn:%d dnode:%d is removed", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); - memset(pVgid, 0, sizeof(SVnodeGid)); - removedNum++; + mInfo("db:%s, vgId:%d, vn:%d dnode:%d, is removed", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); pDnode->numOfVnodes--; - - if (removedNum == 2) goto _OVER; + pVgroup->replica--; + *pDelVgid = *pVgid; + *pVgid = pVgroup->vnodeGid[pVgroup->replica]; + memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid)); + code = 0; + goto _OVER; } } } _OVER: - if (removedNum != 2) return -1; - - for (int32_t vn = 1; vn < TSDB_MAX_REPLICA; ++vn) { - SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; - if (pVgid->dnodeId != 0) { - memcpy(&pVgroup->vnodeGid[0], pVgid, sizeof(SVnodeGid)); - memset(pVgid, 0, sizeof(SVnodeGid)); - } + if (code != 0) { + terrno = TSDB_CODE_APP_ERROR; + mError("db:%s, failed to remove vnode from vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr()); + return -1; } - mInfo("db:%s, vgId:%d, dnode:%d is keeped", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; + mInfo("db:%s, vgId:%d, vn:%d dnode:%d is reserved", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); + } return 0; } @@ -605,23 +600,6 @@ SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) { return epset; } -static int32_t mndProcessCreateVnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - -static int32_t mndProcessAlterVnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - -static int32_t mndProcessDropVnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - -static int32_t mndProcessCompactVnodeRsp(SRpcMsg *pRsp) { return 0; } - static bool mndGetVgroupMaxReplicaFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SVgObj *pVgroup = pObj; int64_t uid = *(int64_t *)p1; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 537f2e0964..4bcd5fc095 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -15,14 +15,15 @@ #include "vnd.h" -static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); -static int vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); -static int vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = 0; @@ -99,11 +100,11 @@ int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) { return code; } -int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) { - void *ptr = NULL; - void *pReq; - int len; - int ret; +int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) { + void *ptr = NULL; + void *pReq; + int32_t len; + int32_t ret; vTrace("vgId:%d, start to process write request %s, version %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); @@ -158,6 +159,9 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg pMsg->contLen - sizeof(SMsgHead)) < 0) { } } break; + case TDMT_VND_ALTER_CONFIRM: + vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp); + break; case TDMT_VND_ALTER_CONFIG: break; default: @@ -190,7 +194,7 @@ _err: return -1; } -int32_t vnodePreprocessQueryMsg(SVnode * pVnode, SRpcMsg * pMsg) { +int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { if (TDMT_VND_QUERY != pMsg->msgType) { return 0; } @@ -212,7 +216,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { } } -int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { +int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { vTrace("message in fetch queue is processing"); char *msgstr = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); @@ -267,7 +271,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { pMetaRsp->precision = pVnode->config.tsdbCfg.precision; } -int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { +int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int32_t ret = TAOS_SYNC_PROPOSE_OTHER_ERROR; if (syncEnvIsStart()) { @@ -357,7 +361,7 @@ int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { return ret; } -static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateStbReq req = {0}; SDecoder coder; @@ -389,9 +393,9 @@ _err: return -1; } -static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SDecoder decoder = {0}; - int rcode = 0; + int32_t rcode = 0; SVCreateTbBatchReq req = {0}; SVCreateTbReq *pCreateReq; SVCreateTbBatchRsp rsp = {0}; @@ -422,7 +426,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, } // loop to create table - for (int iReq = 0; iReq < req.nReqs; iReq++) { + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pCreateReq = req.pReqs + iReq; // validate hash @@ -477,7 +481,7 @@ _exit: return rcode; } -static int vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateStbReq req = {0}; SDecoder dc = {0}; @@ -506,9 +510,9 @@ static int vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, return 0; } -static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SVDropStbReq req = {0}; - int rcode = TSDB_CODE_SUCCESS; + int32_t rcode = TSDB_CODE_SUCCESS; SDecoder decoder = {0}; pRsp->msgType = TDMT_VND_CREATE_STB_RSP; @@ -535,12 +539,12 @@ _exit: return 0; } -static int vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SVAlterTbReq vAlterTbReq = {0}; SVAlterTbRsp vAlterTbRsp = {0}; SDecoder dc = {0}; - int rcode = 0; - int ret; + int32_t rcode = 0; + int32_t ret; SEncoder ec = {0}; STableMetaRsp vMetaRsp = {0}; @@ -582,12 +586,12 @@ _exit: return 0; } -static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SVDropTbBatchReq req = {0}; SVDropTbBatchRsp rsp = {0}; SDecoder decoder = {0}; SEncoder encoder = {0}; - int ret; + int32_t ret; SArray *tbUids = NULL; pRsp->msgType = TDMT_VND_DROP_TABLE_RSP; @@ -609,7 +613,7 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in rsp.pArray = taosArrayInit(req.nReqs, sizeof(SVDropTbRsp)); if (tbUids == NULL || rsp.pArray == NULL) goto _exit; - for (int iReq = 0; iReq < req.nReqs; iReq++) { + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { SVDropTbReq *pDropTbReq = req.pReqs + iReq; SVDropTbRsp dropTbRsp = {0}; @@ -641,7 +645,8 @@ _exit: return 0; } -static int vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSubmitMsgIter *msgIter, const char *tags) { +static int32_t vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSubmitMsgIter *msgIter, + const char *tags) { SSubmitBlkIter blkIter = {0}; STSchema *pSchema = NULL; tb_uid_t suid = 0; @@ -675,7 +680,7 @@ static int vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSub return TSDB_CODE_SUCCESS; } -static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char *tags) { +static int32_t vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char *tags) { ASSERT(pMsg != NULL); SSubmitMsgIter msgIter = {0}; SMeta *pMeta = pVnode->pMeta; @@ -692,7 +697,7 @@ static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char return 0; } -static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; SSubmitRsp submitRsp = {0}; SSubmitMsgIter msgIter = {0}; @@ -808,7 +813,7 @@ _exit: return 0; } -static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateTSmaReq req = {0}; SDecoder coder; @@ -859,3 +864,13 @@ _err: int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen) { return vnodeProcessCreateTSmaReq(pVnode, 1, pCont, contLen, NULL); } + +static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + vInfo("vgId:%d, alter replica confim msg is processed", TD_VID(pVnode)); + pRsp->msgType = TDMT_VND_ALTER_CONFIRM_RSP; + pRsp->code = TSDB_CODE_SUCCESS; + pRsp->pCont = NULL; + pRsp->contLen = 0; + + return 0; +} \ No newline at end of file From 7df1727a7b4bb701d32e4f7a8323c88eb0ffa8ee Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 5 Jun 2022 15:24:57 +0800 Subject: [PATCH 35/63] refactor(query): do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 26 +++---- source/libs/executor/src/executorimpl.c | 6 +- source/libs/executor/src/scanoperator.c | 97 +++++++++++++++---------- 3 files changed, 76 insertions(+), 53 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 061bdc7236..c6220b5ffe 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -303,13 +303,17 @@ typedef struct SScanInfo { int32_t numOfDesc; } SScanInfo; +typedef struct SSampleExecInfo { + double sampleRatio; // data block sample ratio, 1 by default + uint32_t seed; // random seed value +} SSampleExecInfo; + typedef struct STableScanInfo { void* dataReader; SReadHandle readHandle; SFileBlockLoadRecorder readRecorder; int64_t numOfRows; - int64_t elapsedTime; // int32_t prevGroupId; // previous table group id SScanInfo scanInfo; int32_t scanTimes; @@ -330,9 +334,8 @@ typedef struct STableScanInfo { SQueryTableDataCond cond; int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan int32_t dataBlockLoadFlag; - double sampleRatio; // data block sample ratio, 1 by default SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded. - + SSampleExecInfo sample; // sample execution info int32_t curTWinIdx; } STableScanInfo; @@ -692,7 +695,7 @@ void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWin int32_t setGroupResultOutputBuf(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type, int16_t bytes, int32_t groupId, SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, SAggSupporter* pAggSup); void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput); -int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, +int32_t setDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, SArray* pColList); void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, STimeWindow* win); @@ -789,8 +792,6 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol); -void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput); - bool isTaskKilled(SExecTaskInfo* pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); @@ -825,16 +826,14 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result); int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* length); -STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, - SInterval* pInterval, int32_t precision, STimeWindow* win); -int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, - int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, - int32_t order); +STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, + int32_t precision, STimeWindow* win); +int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, + __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey); SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize); -SResultWindowInfo* getSessionTimeWindow(SArray* pWinInfos, TSKEY ts, int64_t gap, - int32_t* pIndex); +SResultWindowInfo* getSessionTimeWindow(SArray* pWinInfos, TSKEY ts, int64_t gap, int32_t* pIndex); int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted); bool functionNeedToExecute(SqlFunctionCtx* pCtx); @@ -842,6 +841,7 @@ int64_t getSmaWaterMark(int64_t interval, double filesFactor); bool isSmaStream(int8_t triggerType); int32_t compareTimeWindow(const void* p1, const void* p2, const void* param); + #ifdef __cplusplus } #endif diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 6434e3a016..6a464de689 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2524,7 +2524,7 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray } } -int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, +int32_t setDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, SArray* pColList) { if (pColList == NULL) { // data from other sources @@ -2679,7 +2679,7 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx } SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; - code = setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, + code = setDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); if (code != 0) { taosMemoryFreeClear(pDataInfo->pRsp); @@ -2803,7 +2803,7 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { SSDataBlock* pRes = pExchangeInfo->pResult; SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; int32_t code = - setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, + setDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); if (pRsp->completed == 1) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8541a0b129..ae6924eaff 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -40,6 +40,22 @@ static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capac static int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, const char* dbName); +static void addTagPseudoColumnData(SReadHandle *pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, SSDataBlock* pBlock); +static bool processBlockWithProbability(const SSampleExecInfo *pInfo); + +bool processBlockWithProbability(const SSampleExecInfo *pInfo) { +#if 0 + if (pInfo->sampleRatio == 1) { + return true; + } + + uint32_t val = taosRandR((uint32_t*) &pInfo->seed); + return (val % ((uint32_t)(1/pInfo->sampleRatio))) == 0; +#else + return true; +#endif +} + static void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { for (int32_t i = 0; i < numOfOutput; ++i) { SWITCH_ORDER(pCtx[i].order); @@ -153,8 +169,6 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn return false; } -static void addTagPseudoColumnData(SReadHandle *pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, SSDataBlock* pBlock); - static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -274,12 +288,9 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction STimeWindow* pTWindow = &pTableScanInfo->cond.twindows[i]; TSWAP(pTWindow->skey, pTWindow->ekey); } - SQueryTableDataCond *pCond = &pTableScanInfo->cond; - taosqsort(pCond->twindows, - pCond->numOfTWindows, - sizeof(STimeWindow), - pCond, - compareTimeWindow); + + SQueryTableDataCond* pCond = &pTableScanInfo->cond; + taosqsort(pCond->twindows, pCond->numOfTWindows, sizeof(STimeWindow), pCond, compareTimeWindow); } void addTagPseudoColumnData(SReadHandle *pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, SSDataBlock* pBlock) { @@ -310,18 +321,19 @@ void addTagPseudoColumnData(SReadHandle *pHandle, SExprInfo* pPseudoExpr, int32_ } else { // these are tags STagVal tagVal = {0}; tagVal.cid = pExpr->base.pParam[0].pCol->colId; - const char *p = metaGetTableTagVal(&mr.me, pColInfoData->info.type, &tagVal); + const char* p = metaGetTableTagVal(&mr.me, pColInfoData->info.type, &tagVal); - char *data = NULL; - if(pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL){ - data = tTagValToData((const STagVal *)p, false); - }else { + char* data = NULL; + if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { + data = tTagValToData((const STagVal*)p, false); + } else { data = (char*)p; } for (int32_t i = 0; i < pBlock->info.rows; ++i) { colDataAppend(pColInfoData, i, data, (data == NULL)); } + if (data && (pColInfoData->info.type != TSDB_DATA_TYPE_JSON) && p != NULL && IS_VAR_DATA_TYPE(((const STagVal*)p)->type)) { taosMemoryFree(data); @@ -359,6 +371,12 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } + // process this data block based on the probabilities + bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample); + if (!processThisBlock) { + continue; + } + tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBlock->info); uint32_t status = 0; @@ -433,6 +451,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STimeWindow* pWin = &pTableScanInfo->cond.twindows[i]; qDebug("%s\t qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); } + while (pTableScanInfo->scanTimes < total) { while (pTableScanInfo->curTWinIdx < pTableScanInfo->cond.numOfTWindows) { SSDataBlock* p = doTableScanImpl(pOperator); @@ -502,11 +521,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; + goto _error; } SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc; @@ -517,7 +532,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { - return NULL; + goto _error; } if (pTableScanNode->scan.pScanPseudoCols != NULL) { @@ -528,32 +543,40 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; // pInfo->scanInfo = (SScanInfo){.numOfAsc = 0, .numOfDesc = 1}; // for debug purpose - pInfo->readHandle = *readHandle; - pInfo->interval = extractIntervalInfo(pTableScanNode); - pInfo->sampleRatio = pTableScanNode->ratio; - pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; - pInfo->pResBlock = createResDataBlock(pDescNode); - pInfo->pFilterNode = pTableScanNode->scan.node.pConditions; - pInfo->dataReader = pDataReader; - pInfo->scanFlag = MAIN_SCAN; - pInfo->pColMatchInfo = pColList; - pInfo->curTWinIdx = 0; + pInfo->readHandle = *readHandle; + pInfo->interval = extractIntervalInfo(pTableScanNode); + pInfo->sample.sampleRatio= pTableScanNode->ratio; + pInfo->sample.seed = taosGetTimestampMs(); - pOperator->name = "TableScanOperator"; // for debug purpose + pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; + pInfo->pResBlock = createResDataBlock(pDescNode); + pInfo->pFilterNode = pTableScanNode->scan.node.pConditions; + pInfo->dataReader = pDataReader; + pInfo->scanFlag = MAIN_SCAN; + pInfo->pColMatchInfo = pColList; + pInfo->curTWinIdx = 0; + + pOperator->name = "TableScanOperator"; // for debug purpose pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfExprs = numOfCols; - pOperator->pTaskInfo = pTaskInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfExprs = numOfCols; + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, NULL, destroyTableScanOperatorInfo, NULL, NULL, getTableScannerExecInfo); // for non-blocking operator, the open cost is always 0 pOperator->cost.openCost = 0; - return pOperator; + + _error: + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; } SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* pTaskInfo) { @@ -1351,7 +1374,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } SRetrieveMetaTableRsp* pTableRsp = pInfo->pRsp; - setSDataBlockFromFetchRsp(pInfo->pRes, &pInfo->loadInfo, pTableRsp->numOfRows, pTableRsp->data, + setDataBlockFromFetchRsp(pInfo->pRes, &pInfo->loadInfo, pTableRsp->numOfRows, pTableRsp->data, pTableRsp->compLen, pOperator->numOfExprs, startTs, NULL, pInfo->scanCols); // todo log the filter info From 6bd6822637b56adb631833018e418779a5fbd3da Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 5 Jun 2022 15:26:25 +0800 Subject: [PATCH 36/63] refactor: remove unused functions. --- source/libs/executor/src/executorimpl.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 6a464de689..c8f49d5dd0 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2723,17 +2723,6 @@ _error: return NULL; } -static SSDataBlock* concurrentlyLoadRemoteData(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - if (pOperator->status == OP_RES_TO_RETURN) { - return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); - } else { - ASSERT(0); - } -} - static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; From e25560b64086d05704a663ae6d437664800e9aa8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 5 Jun 2022 15:35:34 +0800 Subject: [PATCH 37/63] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ae6924eaff..eadfd9be88 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -546,7 +546,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->readHandle = *readHandle; pInfo->interval = extractIntervalInfo(pTableScanNode); pInfo->sample.sampleRatio= pTableScanNode->ratio; - pInfo->sample.seed = taosGetTimestampMs(); + pInfo->sample.seed = taosGetTimestampSec(); pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; pInfo->pResBlock = createResDataBlock(pDescNode); From 4391ed4af6666992dfbf6adf323931a6ad0b322e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 5 Jun 2022 15:37:04 +0800 Subject: [PATCH 38/63] test:update the test suit. --- tests/script/tsim/testsuit.sim | 64 +++++++++++++++++----------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/tests/script/tsim/testsuit.sim b/tests/script/tsim/testsuit.sim index b2ca1e0345..c73efde9d5 100644 --- a/tests/script/tsim/testsuit.sim +++ b/tests/script/tsim/testsuit.sim @@ -3,8 +3,8 @@ #run tsim/user/privilege2.sim #run tsim/user/user_len.sim #run tsim/user/privilege1.sim -##run tsim/user/pass_len.sim -##run tsim/table/basic1.sim +#run tsim/user/pass_len.sim +#run tsim/table/basic1.sim #run tsim/trans/lossdata1.sim #run tsim/trans/create_db.sim ##run tsim/stable/alter_metrics.sim @@ -13,20 +13,20 @@ ##run tsim/stable/column_drop.sim ##run tsim/stable/column_modify.sim ##run tsim/stable/tag_rename.sim -##run tsim/stable/vnode3.sim -##run tsim/stable/metrics.sim +#run tsim/stable/vnode3.sim +#run tsim/stable/metrics.sim ##run tsim/stable/alter_insert2.sim -##run tsim/stable/show.sim -##run tsim/stable/alter_import.sim +#run tsim/stable/show.sim +#run tsim/stable/alter_import.sim ##run tsim/stable/tag_add.sim -##run tsim/stable/tag_drop.sim -##run tsim/stable/column_add.sim -##run tsim/stable/alter_count.sim -##run tsim/stable/values.sim -##run tsim/stable/dnode3.sim -##run tsim/stable/alter_insert1.sim -##run tsim/stable/refcount.sim -##run tsim/stable/disk.sim +#run tsim/stable/tag_drop.sim +#run tsim/stable/column_add.sim +#run tsim/stable/alter_count.sim +#run tsim/stable/values.sim +#run tsim/stable/dnode3.sim +#run tsim/stable/alter_insert1.sim +#run tsim/stable/refcount.sim +#run tsim/stable/disk.sim #run tsim/db/basic1.sim #run tsim/db/basic3.sim ##run tsim/db/basic7.sim @@ -40,24 +40,24 @@ #run tsim/mnode/basic3.sim #run tsim/mnode/basic2.sim #run tsim/parser/fourArithmetic-basic.sim -run tsim/parser/groupby-basic.sim -run tsim/snode/basic1.sim -run tsim/query/time_process.sim -run tsim/query/stddev.sim -run tsim/query/interval-offset.sim -run tsim/query/charScalarFunction.sim -run tsim/query/complex_select.sim -run tsim/query/explain.sim -run tsim/query/crash_sql.sim -run tsim/query/diff.sim -run tsim/query/complex_limit.sim -run tsim/query/complex_having.sim -run tsim/query/udf.sim -run tsim/query/complex_group.sim -run tsim/query/interval.sim -run tsim/query/session.sim -run tsim/query/scalarFunction.sim -run tsim/query/scalarNull.sim +#run tsim/parser/groupby-basic.sim +#run tsim/snode/basic1.sim +#run tsim/query/time_process.sim +#run tsim/query/stddev.sim +#run tsim/query/interval-offset.sim +#run tsim/query/charScalarFunction.sim +#run tsim/query/complex_select.sim +#run tsim/query/explain.sim +#run tsim/query/crash_sql.sim +#run tsim/query/diff.sim +#run tsim/query/complex_limit.sim +#run tsim/query/complex_having.sim +#run tsim/query/udf.sim +#run tsim/query/complex_group.sim +#run tsim/query/interval.sim +#run tsim/query/session.sim +#run tsim/query/scalarFunction.sim +##run tsim/query/scalarNull.sim run tsim/query/complex_where.sim run tsim/tmq/basic1.sim run tsim/tmq/basic4.sim From 5a8591052d805c9af16b3c974c88e392a71edc64 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 5 Jun 2022 07:46:57 +0000 Subject: [PATCH 39/63] more work --- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 201 +++++++++++++++++----- 1 file changed, 156 insertions(+), 45 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 4948d8a0b1..6c24396810 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -19,20 +19,18 @@ typedef struct { SMemTable *pMemTable; int32_t minutes; int8_t precision; - int32_t sfid; - int32_t efid; + TSKEY nCommitKey; SReadH readh; SDFileSet wSet; - SArray *aDelInfo; SArray *aBlkIdx; SArray *aSupBlk; SArray *aSubBlk; + SArray *aDelInfo; } SCommitH; -static int32_t tsdbStartCommit(SCommitH *pCHandle, STsdb *pTsdb); -static int32_t tsdbEndCommit(SCommitH *pCHandle); -static int32_t tsdbCommitToFile(SCommitH *pCHandle, int32_t fid); -static int32_t tsdbCommitDelete(SCommitH *pCHandle); +static int32_t tsdbCommitStart(SCommitH *pCHandle, STsdb *pTsdb); +static int32_t tsdbCommitEnd(SCommitH *pCHandle); +static int32_t tsdbCommitImpl(SCommitH *pCHandle); int32_t tsdbBegin2(STsdb *pTsdb) { int32_t code = 0; @@ -53,26 +51,19 @@ int32_t tsdbCommit2(STsdb *pTsdb) { SCommitH ch = {0}; // start to commit - code = tsdbStartCommit(&ch, pTsdb); + code = tsdbCommitStart(&ch, pTsdb); if (code) { goto _exit; } // commit - for (int32_t fid = ch.sfid; fid <= ch.efid; fid++) { - code = tsdbCommitToFile(&ch, fid); - if (code) { - goto _err; - } - } - - code = tsdbCommitDelete(&ch); + code = tsdbCommitImpl(&ch); if (code) { goto _err; } // end commit - code = tsdbEndCommit(&ch); + code = tsdbCommitEnd(&ch); if (code) { goto _exit; } @@ -81,11 +72,11 @@ _exit: return code; _err: - // TODO: rollback + tsdbError("vgId:%d failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } -static int32_t tsdbStartCommit(SCommitH *pCHandle, STsdb *pTsdb) { +static int32_t tsdbCommitStart(SCommitH *pCHandle, STsdb *pTsdb) { int32_t code = 0; SMemTable *pMemTable = (SMemTable *)pTsdb->mem; @@ -100,28 +91,32 @@ static int32_t tsdbStartCommit(SCommitH *pCHandle, STsdb *pTsdb) { pCHandle->pMemTable = pMemTable; pCHandle->minutes = pTsdb->keepCfg.days; pCHandle->precision = pTsdb->keepCfg.precision; - pCHandle->sfid = TSDB_KEY_FID(pMemTable->minKey.ts, pCHandle->minutes, pCHandle->precision); - pCHandle->efid = TSDB_KEY_FID(pMemTable->maxKey.ts, pCHandle->minutes, pCHandle->precision); + pCHandle->nCommitKey = pMemTable->minKey.ts; code = tsdbInitReadH(&pCHandle->readh, pTsdb); if (code) { goto _err; } - pCHandle->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx)); + pCHandle->aBlkIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pCHandle->aBlkIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - pCHandle->aSupBlk = taosArrayInit(1024, sizeof(SBlock)); + pCHandle->aSupBlk = taosArrayInit(0, sizeof(SBlock)); if (pCHandle->aSupBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - pCHandle->aSubBlk = taosArrayInit(1024, sizeof(SBlock)); + pCHandle->aSubBlk = taosArrayInit(0, sizeof(SBlock)); if (pCHandle->aSubBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } + pCHandle->aDelInfo = taosArrayInit(0, sizeof(SDelInfo)); + if (pCHandle->aDelInfo == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } // start FS transaction tsdbStartFSTxn(pTsdb, 0, 0); @@ -132,7 +127,7 @@ _err: return code; } -static int32_t tsdbEndCommit(SCommitH *pCHandle) { +static int32_t tsdbCommitEnd(SCommitH *pCHandle) { int32_t code = 0; STsdb *pTsdb = pCHandle->pMemTable->pTsdb; SMemTable *pMemTable = (SMemTable *)pTsdb->imem; @@ -144,6 +139,7 @@ static int32_t tsdbEndCommit(SCommitH *pCHandle) { } // close handle + taosArrayClear(pCHandle->aDelInfo); taosArrayClear(pCHandle->aSubBlk); taosArrayClear(pCHandle->aSupBlk); taosArrayClear(pCHandle->aBlkIdx); @@ -160,10 +156,29 @@ _err: return code; } -static int32_t tsdbCommitTableData(SCommitH *pCHandle, SMemData *pMemData, SBlockIdx *pBlockIdx) { +static int32_t tsdbCommitTableStart(SCommitH *pCHandle) { + int32_t code = 0; + // TODO + return code; +} + +static int32_t tsdbCommitTableEnd(SCommitH *pCHandle) { + int32_t code = 0; + // TODO + return code; +} + +static int32_t tsdbCommitTable(SCommitH *pCHandle, SMemData *pMemData, SBlockIdx *pBlockIdx) { int32_t code = 0; SMemDataIter iter = {0}; + // commit table start + code = tsdbCommitTableStart(pCHandle); + if (code) { + goto _err; + } + + // commit table impl if (pMemData && pBlockIdx) { // merge } else if (pMemData) { @@ -172,6 +187,15 @@ static int32_t tsdbCommitTableData(SCommitH *pCHandle, SMemData *pMemData, SBloc // save old ones } + // commit table end + code = tsdbCommitTableEnd(pCHandle); + if (code) { + goto _err; + } + + return code; + +_err: return code; } @@ -193,38 +217,58 @@ static int32_t tsdbTableIdCmprFn(const void *p1, const void *p2) { return 0; } -static int32_t tsdbCommitToFile(SCommitH *pCHandle, int32_t fid) { + +static int32_t tsdbWriteBlockIdx(SDFile *pFile, SArray *pArray, uint8_t **ppBuf) { + int32_t code = 0; + // TODO + return code; +} + +static int32_t tsdbCommitFileStart(SCommitH *pCHandle, int32_t fid) { + int32_t code = 0; + // TODO + return code; +} + +static int32_t tsdbCommitFileEnd(SCommitH *pCHandle) { + int32_t code = 0; + // TODO + return code; +} + +static int32_t tsdbCommitFile(SCommitH *pCHandle, int32_t fid) { int32_t code = 0; SMemDataIter iter = {0}; TSDBROW *pRow = NULL; int8_t hasData = 0; TSKEY fidSKey; TSKEY fidEKey; - int32_t iMemData = 0; - int32_t nMemData = taosArrayGetSize(pCHandle->pMemTable->aMemData); - int32_t iBlockIdx = 0; + int32_t iMemData; + int32_t nMemData; + int32_t iBlockIdx; int32_t nBlockIdx; - // check if there are data in the time range - for (; iMemData < nMemData; iMemData++) { - SMemData *pMemData = (SMemData *)taosArrayGetP(pCHandle->pMemTable->aMemData, iMemData); - tsdbMemDataIterOpen(pMemData, &(TSDBKEY){.ts = fidSKey, .version = 0}, 0, &iter); - tsdbMemDataIterGet(&iter, &pRow); - - if (pRow->tsRow.ts >= fidSKey && pRow->tsRow.ts <= fidEKey) { - hasData = 1; - break; - } - } - - if (!hasData) return code; + pCHandle->nCommitKey = TSKEY_MAX; // create or open the file to commit(todo) + code = tsdbCommitFileStart(pCHandle, fid); + if (code) { + goto _err; + } // loop to commit each table data + iMemData = 0; + nMemData = taosArrayGetSize(pCHandle->pMemTable->aMemData); + iBlockIdx = 0; nBlockIdx = 0; for (;;) { - if (iBlockIdx >= nBlockIdx && iMemData >= nMemData) break; + if (iBlockIdx >= nBlockIdx && iMemData >= nMemData) { + // code = tsdbWriteBlockIdx(); + // if (code) { + // goto _err; + // } + break; + } SMemData *pMemData = NULL; SBlockIdx *pBlockIdx = NULL; @@ -256,12 +300,42 @@ static int32_t tsdbCommitToFile(SCommitH *pCHandle, int32_t fid) { } } - code = tsdbCommitTableData(pCHandle, pMemData, pBlockIdx); + code = tsdbCommitTable(pCHandle, pMemData, pBlockIdx); if (code) { goto _err; } } + // close file + code = tsdbCommitFileEnd(pCHandle); + if (code) { + goto _err; + } + + return code; + +_err: + return code; +} + +static int32_t tsdbCommitData(SCommitH *pCHandle) { + int32_t code = 0; + int32_t fid; + + if (pCHandle->pMemTable->nRows == 0) goto _exit; + + // loop to commit to each file + for (;;) { + if (pCHandle->nCommitKey == TSKEY_MAX) break; + + fid = TSDB_KEY_FID(pCHandle->nCommitKey, pCHandle->minutes, pCHandle->precision); + code = tsdbCommitFile(pCHandle, fid); + if (code) { + goto _err; + } + } + +_exit: return code; _err: @@ -320,9 +394,46 @@ static int32_t tsdbCommitDelete(SCommitH *pCHandle) { taosArraySort(pCHandle->aDelInfo, delInfoCmprFn); + // write to new file + _exit: return code; +_err: + return code; +} + +static int32_t tsdbCommitCache(SCommitH *pCHandle) { + int32_t code = 0; + // TODO + return code; +} + +static int32_t tsdbCommitImpl(SCommitH *pCHandle) { + int32_t code = 0; + + // commit data + code = tsdbCommitData(pCHandle); + if (code) { + goto _err; + } + + // commit delete + code = tsdbCommitDelete(pCHandle); + if (code) { + goto _err; + } + + // commit cache if need (todo) + if (0) { + code = tsdbCommitCache(pCHandle); + if (code) { + goto _err; + } + } + + return code; + _err: return code; } \ No newline at end of file From 55d41c8f5c6fc0be2b96507d26319b51a00de81e Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 5 Jun 2022 15:48:06 +0800 Subject: [PATCH 40/63] feat: sql command 'delete from' --- include/libs/nodes/nodes.h | 6 +- include/libs/nodes/plannodes.h | 118 ++- source/libs/command/src/explain.c | 22 +- source/libs/executor/src/executorimpl.c | 16 +- source/libs/executor/src/timewindowoperator.c | 948 +++++++++--------- source/libs/nodes/src/nodesCloneFuncs.c | 17 +- source/libs/nodes/src/nodesCodeFuncs.c | 203 +++- source/libs/nodes/src/nodesTraverseFuncs.c | 4 +- source/libs/nodes/src/nodesUtilFuncs.c | 19 +- source/libs/parser/src/parAstCreater.c | 16 + source/libs/parser/src/parTranslater.c | 3 +- source/libs/parser/test/parInitialDTest.cpp | 4 + source/libs/planner/src/planLogicCreater.c | 131 ++- source/libs/planner/src/planOptimizer.c | 6 +- source/libs/planner/src/planPhysiCreater.c | 164 +-- source/libs/planner/src/planScaleOut.c | 71 +- source/libs/planner/src/planSpliter.c | 71 +- source/libs/planner/test/planOtherTest.cpp | 12 + 18 files changed, 1089 insertions(+), 742 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 985bc63f42..29fabb5418 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -188,7 +188,7 @@ typedef enum ENodeType { QUERY_NODE_LOGIC_PLAN_JOIN, QUERY_NODE_LOGIC_PLAN_AGG, QUERY_NODE_LOGIC_PLAN_PROJECT, - QUERY_NODE_LOGIC_PLAN_VNODE_MODIF, + QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY, QUERY_NODE_LOGIC_PLAN_EXCHANGE, QUERY_NODE_LOGIC_PLAN_MERGE, QUERY_NODE_LOGIC_PLAN_WINDOW, @@ -210,7 +210,8 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, QUERY_NODE_PHYSICAL_PLAN_MERGE, QUERY_NODE_PHYSICAL_PLAN_SORT, - QUERY_NODE_PHYSICAL_PLAN_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL, @@ -223,6 +224,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_PARTITION, QUERY_NODE_PHYSICAL_PLAN_DISPATCH, QUERY_NODE_PHYSICAL_PLAN_INSERT, + QUERY_NODE_PHYSICAL_PLAN_DELETE, QUERY_NODE_PHYSICAL_SUBPLAN, QUERY_NODE_PHYSICAL_PLAN } ENodeType; diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index e11930676e..38c3f150ef 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -37,29 +37,30 @@ typedef struct SLogicNode { typedef enum EScanType { SCAN_TYPE_TAG = 1, SCAN_TYPE_TABLE, SCAN_TYPE_SYSTEM_TABLE, SCAN_TYPE_STREAM } EScanType; typedef struct SScanLogicNode { - SLogicNode node; - SNodeList* pScanCols; - SNodeList* pScanPseudoCols; - struct STableMeta* pMeta; - SVgroupsInfo* pVgroupList; - EScanType scanType; - uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count - STimeWindow scanRange; - SName tableName; - bool showRewrite; - double ratio; - SNodeList* pDynamicScanFuncs; - int32_t dataRequired; - int64_t interval; - int64_t offset; - int64_t sliding; - int8_t intervalUnit; - int8_t slidingUnit; - SNode* pTagCond; - int8_t triggerType; - int64_t watermark; - int16_t tsColId; - double filesFactor; + SLogicNode node; + SNodeList* pScanCols; + SNodeList* pScanPseudoCols; + int8_t tableType; + uint64_t tableId; + SVgroupsInfo* pVgroupList; + EScanType scanType; + uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count + STimeWindow scanRange; + SName tableName; + bool showRewrite; + double ratio; + SNodeList* pDynamicScanFuncs; + int32_t dataRequired; + int64_t interval; + int64_t offset; + int64_t sliding; + int8_t intervalUnit; + int8_t slidingUnit; + SNode* pTagCond; + int8_t triggerType; + int64_t watermark; + int16_t tsColId; + double filesFactor; } SScanLogicNode; typedef struct SJoinLogicNode { @@ -85,12 +86,20 @@ typedef struct SProjectLogicNode { int64_t soffset; } SProjectLogicNode; -typedef struct SVnodeModifLogicNode { - SLogicNode node; - int32_t msgType; - SArray* pDataBlocks; - SVgDataBlocks* pVgDataBlocks; -} SVnodeModifLogicNode; +typedef enum EModifyTableType { MODIFY_TABLE_TYPE_INSERT = 1, MODIFY_TABLE_TYPE_DELETE } EModifyTableType; + +typedef struct SVnodeModifyLogicNode { + SLogicNode node; + EModifyTableType modifyType; + int32_t msgType; + SArray* pDataBlocks; + SVgDataBlocks* pVgDataBlocks; + SNode* pModifyRows; // SColumnNode + uint64_t tableId; + int8_t tableType; // table type + char tableFName[TSDB_TABLE_FNAME_LEN]; + STimeWindow deleteTimeRange; +} SVnodeModifyLogicNode; typedef struct SExchangeLogicNode { SLogicNode node; @@ -106,28 +115,30 @@ typedef struct SMergeLogicNode { typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType; -typedef enum EStreamIntervalAlgorithm { - STREAM_INTERVAL_ALGO_FINAL = 1, - STREAM_INTERVAL_ALGO_SEMI, - STREAM_INTERVAL_ALGO_SINGLE -} EStreamIntervalAlgorithm; +typedef enum EIntervalAlgorithm { + INTERVAL_ALGO_HASH = 1, + INTERVAL_ALGO_SORT_MERGE, + INTERVAL_ALGO_STREAM_FINAL, + INTERVAL_ALGO_STREAM_SEMI, + INTERVAL_ALGO_STREAM_SINGLE, +} EIntervalAlgorithm; typedef struct SWindowLogicNode { - SLogicNode node; - EWindowType winType; - SNodeList* pFuncs; - int64_t interval; - int64_t offset; - int64_t sliding; - int8_t intervalUnit; - int8_t slidingUnit; - int64_t sessionGap; - SNode* pTspk; - SNode* pStateExpr; - int8_t triggerType; - int64_t watermark; - double filesFactor; - EStreamIntervalAlgorithm stmInterAlgo; + SLogicNode node; + EWindowType winType; + SNodeList* pFuncs; + int64_t interval; + int64_t offset; + int64_t sliding; + int8_t intervalUnit; + int8_t slidingUnit; + int64_t sessionGap; + SNode* pTspk; + SNode* pStateExpr; + int8_t triggerType; + int64_t watermark; + double filesFactor; + EIntervalAlgorithm intervalAlgo; } SWindowLogicNode; typedef struct SFillLogicNode { @@ -308,6 +319,7 @@ typedef struct SIntervalPhysiNode { int8_t slidingUnit; } SIntervalPhysiNode; +typedef SIntervalPhysiNode SSortMergeIntervalPhysiNode; typedef SIntervalPhysiNode SStreamIntervalPhysiNode; typedef SIntervalPhysiNode SStreamFinalIntervalPhysiNode; typedef SIntervalPhysiNode SStreamSemiIntervalPhysiNode; @@ -370,6 +382,14 @@ typedef struct SDataInserterNode { char* pData; } SDataInserterNode; +typedef struct SDataDeleterNode { + SDataSinkNode sink; + uint64_t tableId; + int8_t tableType; // table type + char tableFName[TSDB_TABLE_FNAME_LEN]; + STimeWindow deleteTimeRange; +} SDataDeleterNode; + typedef struct SSubplan { ENodeType type; SSubplanId id; // unique id of the subplan diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 831b7017b2..98ae5e2fb1 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -153,7 +153,7 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo pPhysiChildren = pSortNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: { + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: { SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; pPhysiChildren = pIntNode->window.node.pChildren; break; @@ -164,12 +164,12 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo break; } case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: { - SStateWinodwPhysiNode* pStateNode = (SStateWinodwPhysiNode*) pNode; + SStateWinodwPhysiNode *pStateNode = (SStateWinodwPhysiNode *)pNode; pPhysiChildren = pStateNode->window.node.pChildren; break; } case QUERY_NODE_PHYSICAL_PLAN_PARTITION: { - SPartitionPhysiNode* partitionPhysiNode = (SPartitionPhysiNode*) pNode; + SPartitionPhysiNode *partitionPhysiNode = (SPartitionPhysiNode *)pNode; pPhysiChildren = partitionPhysiNode->node.pChildren; break; } @@ -397,7 +397,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i int32_t nodeNum = taosArrayGetSize(pResNode->pExecInfo); for (int32_t i = 0; i < nodeNum; ++i) { - SExplainExecInfo * execInfo = taosArrayGet(pResNode->pExecInfo, i); + SExplainExecInfo *execInfo = taosArrayGet(pResNode->pExecInfo, i); STableScanAnalyzeInfo *pScanInfo = (STableScanAnalyzeInfo *)execInfo->verboseInfo; EXPLAIN_ROW_APPEND("total_blocks=%d", pScanInfo->totalBlocks); @@ -429,7 +429,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, + pTblScanNode->scanRange.ekey); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); @@ -641,7 +642,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); } - SDataBlockDescNode* pDescNode = pSortNode->node.pOutputDataBlockDesc; + SDataBlockDescNode *pDescNode = pSortNode->node.pOutputDataBlockDesc; EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pDescNode->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pDescNode->totalRowSize); @@ -667,7 +668,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i int32_t nodeNum = taosArrayGetSize(pResNode->pExecInfo); SExplainExecInfo *execInfo = taosArrayGet(pResNode->pExecInfo, 0); - SSortExecInfo * pExecInfo = (SSortExecInfo *)execInfo->verboseInfo; + SSortExecInfo *pExecInfo = (SSortExecInfo *)execInfo->verboseInfo; EXPLAIN_ROW_APPEND("%s", pExecInfo->sortMethod == SORT_QSORT_T ? "quicksort" : "merge sort"); if (pExecInfo->sortBuffer > 1024 * 1024) { EXPLAIN_ROW_APPEND(" Buffers:%.2f Mb", pExecInfo->sortBuffer / (1024 * 1024.0)); @@ -701,7 +702,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: { + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: { SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->window.pTspk)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); @@ -784,7 +785,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: { SStateWinodwPhysiNode *pStateNode = (SStateWinodwPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_STATE_WINDOW_FORMAT, nodesGetNameFromColumnNode(((STargetNode*)pStateNode->pStateKey)->pExpr)); + EXPLAIN_ROW_NEW(level, EXPLAIN_STATE_WINDOW_FORMAT, + nodesGetNameFromColumnNode(((STargetNode *)pStateNode->pStateKey)->pExpr)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); @@ -823,7 +825,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i case QUERY_NODE_PHYSICAL_PLAN_PARTITION: { SPartitionPhysiNode *pPartNode = (SPartitionPhysiNode *)pNode; - SNode* p = nodesListGetNode(pPartNode->pPartitionKeys, 0); + SNode *p = nodesListGetNode(pPartNode->pPartitionKeys, 0); EXPLAIN_ROW_NEW(level, EXPLAIN_PARITION_FORMAT, nodesGetNameFromColumnNode(p)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 07f4251340..b6d8059f1a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4052,12 +4052,12 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p goto _error; } - pInfo->limit = *pLimit; - pInfo->slimit = *pSlimit; - pInfo->curOffset = pLimit->offset; + pInfo->limit = *pLimit; + pInfo->slimit = *pSlimit; + pInfo->curOffset = pLimit->offset; pInfo->curSOffset = pSlimit->offset; pInfo->binfo.pRes = pResBlock; - pInfo->pFilterNode= pCondition; + pInfo->pFilterNode = pCondition; int32_t numOfCols = num; int32_t numOfRows = 4096; @@ -4487,7 +4487,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); SLimit limit = {.limit = pProjPhyNode->limit, .offset = pProjPhyNode->offset}; SLimit slimit = {.limit = pProjPhyNode->slimit, .offset = pProjPhyNode->soffset}; - pOptr = createProjectOperatorInfo(ops[0], pExprInfo, num, pResBlock, &limit, &slimit, pProjPhyNode->node.pConditions, pTaskInfo); + pOptr = createProjectOperatorInfo(ops[0], pExprInfo, num, pResBlock, &limit, &slimit, + pProjPhyNode->node.pConditions, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_AGG == type) { SAggPhysiNode* pAggNode = (SAggPhysiNode*)pPhyNode; SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); @@ -4507,7 +4508,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pScalarExprInfo, numOfScalarExpr, pTaskInfo); } - } else if (QUERY_NODE_PHYSICAL_PLAN_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { + } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); @@ -5187,8 +5188,7 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo return TSDB_CODE_SUCCESS; } -int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, - size_t size) { +int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, size_t size) { pSup->keySize = sizeof(int64_t) + sizeof(TSKEY); pSup->pKeyBuf = taosMemoryCalloc(1, pSup->keySize); pSup->pResultRows = taosArrayInit(1024, size); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index ce3ed74b3b..b621d729e0 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1,5 +1,5 @@ -#include "function.h" #include "executorimpl.h" +#include "function.h" #include "functionMgt.h" #include "tdatablock.h" #include "ttime.h" @@ -30,9 +30,7 @@ static void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, S // do nothing } -static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols) { - return tsCols == NULL? win->skey:tsCols[0]; -} +static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols) { return tsCols == NULL ? win->skey : tsCols[0]; } static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, bool ascQuery) { @@ -55,8 +53,8 @@ static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, T } // get the correct time window according to the handled timestamp -STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, - SInterval* pInterval, int32_t precision, STimeWindow* win) { +STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, + int32_t precision, STimeWindow* win) { STimeWindow w = {0}; if (pResultRowInfo->cur.pageId == -1) { // the first window, from the previous stored value @@ -155,14 +153,14 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se forwardRows += 1; } } -// int32_t end = searchFn((char*)pData, pos + 1, ekey, order); -// if (end >= 0) { -// forwardRows = pos - end; -// -// if (pData[end] == ekey) { -// forwardRows += 1; -// } -// } + // int32_t end = searchFn((char*)pData, pos + 1, ekey, order); + // if (end >= 0) { + // forwardRows = pos - end; + // + // if (pData[end] == ekey) { + // forwardRows += 1; + // } + // } } assert(forwardRows >= 0); @@ -240,9 +238,8 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { return midPos; } -int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, - TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, - int32_t order) { +int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, + __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) { assert(startPos >= 0 && startPos < pDataBlockInfo->rows); int32_t num = -1; @@ -311,25 +308,24 @@ static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t o tw->ekey -= 1; } -void doTimeWindowInterpolation(SIntervalAggOperatorInfo *pInfo, int32_t numOfExprs, SArray* pDataBlock, TSKEY prevTs, +void doTimeWindowInterpolation(SIntervalAggOperatorInfo* pInfo, int32_t numOfExprs, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { SqlFunctionCtx* pCtx = pInfo->binfo.pCtx; int32_t index = 1; for (int32_t k = 0; k < numOfExprs; ++k) { - // todo use flag instead of function name if (strcmp(pCtx[k].pExpr->pExpr->_function.functionName, "twa") != 0) { pCtx[k].start.key = INT64_MIN; continue; } -// if (functionId != FUNCTION_TWA && functionId != FUNCTION_INTERP) { -// pCtx[k].start.key = INT64_MIN; -// continue; -// } + // if (functionId != FUNCTION_TWA && functionId != FUNCTION_INTERP) { + // pCtx[k].start.key = INT64_MIN; + // continue; + // } - SFunctParam* pParam = &pCtx[k].param[0]; + SFunctParam* pParam = &pCtx[k].param[0]; SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, pParam->pCol->slotId); ASSERT(pColInfo->info.colId == pParam->pCol->colId && curTs != windowKey); @@ -366,26 +362,25 @@ void doTimeWindowInterpolation(SIntervalAggOperatorInfo *pInfo, int32_t numOfExp } else if (functionId == FUNCTION_TWA) { #endif - SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; - SPoint point2 = (SPoint){.key = curTs, .val = &v2}; - SPoint point = (SPoint){.key = windowKey, .val = &v}; + SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; + SPoint point2 = (SPoint){.key = curTs, .val = &v2}; + SPoint point = (SPoint){.key = windowKey, .val = &v}; - taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); - if (type == RESULT_ROW_START_INTERP) { - pCtx[k].start.key = point.key; - pCtx[k].start.val = v; - } else { - pCtx[k].end.key = point.key; - pCtx[k].end.val = v; - } - - index += 1; + if (type == RESULT_ROW_START_INTERP) { + pCtx[k].start.key = point.key; + pCtx[k].start.val = v; + } else { + pCtx[k].end.key = point.key; + pCtx[k].end.val = v; } + + index += 1; + } #if 0 } #endif - } static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { @@ -400,14 +395,14 @@ static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, in } } -static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo *pInfo, SqlFunctionCtx* pCtx, int32_t numOfExprs, int32_t pos, - SSDataBlock* pBlock, const TSKEY* tsCols, STimeWindow* win) { - bool ascQuery = (pInfo->order == TSDB_ORDER_ASC); +static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfExprs, + int32_t pos, SSDataBlock* pBlock, const TSKEY* tsCols, STimeWindow* win) { + bool ascQuery = (pInfo->order == TSDB_ORDER_ASC); TSKEY curTs = tsCols[pos]; SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0); - TSKEY lastTs = *(int64_t*) pTsKey->pData; + TSKEY lastTs = *(int64_t*)pTsKey->pData; // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. // start exactly from this point, no need to do interpolation @@ -429,8 +424,9 @@ static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo *pInfo, S return true; } -static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo *pInfo, SqlFunctionCtx* pCtx, int32_t numOfExprs, int32_t endRowIndex, - SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { +static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfExprs, + int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, + TSKEY blockEkey, STimeWindow* win) { int32_t order = pInfo->order; TSKEY actualEndKey = tsCols[endRowIndex]; @@ -452,14 +448,14 @@ static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo *pInfo, Sql assert(nextRowIndex >= 0); TSKEY nextKey = tsCols[nextRowIndex]; - doTimeWindowInterpolation(pInfo, numOfExprs, pDataBlock, actualEndKey, endRowIndex, nextKey, - nextRowIndex, key, RESULT_ROW_END_INTERP); + doTimeWindowInterpolation(pInfo, numOfExprs, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key, + RESULT_ROW_END_INTERP); return true; } static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, TSKEY* primaryKeys, int32_t prevPosition, int32_t order) { - bool ascQuery = (order == TSDB_ORDER_ASC); + bool ascQuery = (order == TSDB_ORDER_ASC); int32_t precision = pInterval->precision; getNextTimeWindow(pInterval, precision, order, pNext); @@ -542,8 +538,9 @@ static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { } } -static void doWindowBorderInterpolation(SIntervalAggOperatorInfo *pInfo, SSDataBlock* pBlock, int32_t numOfExprs, SqlFunctionCtx* pCtx, - SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardRows) { +static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t numOfExprs, + SqlFunctionCtx* pCtx, SResultRow* pResult, STimeWindow* win, int32_t startPos, + int32_t forwardRows) { if (!pInfo->timeWindowInterpo) { return; } @@ -600,7 +597,7 @@ static void saveDataBlockLastRow(SArray* pPrevKeys, const SSDataBlock* pBlock, S SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pc->slotId); SGroupKeys* pkey = taosArrayGet(pPrevKeys, k); - for(int32_t i = pBlock->info.rows - 1; i >= 0; --i) { + for (int32_t i = pBlock->info.rows - 1; i >= 0; --i) { if (colDataIsNull_s(pColInfo, i)) { continue; } @@ -642,24 +639,26 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId); if (pr->closed) { - ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) && isResultRowInterpolated(pr, RESULT_ROW_END_INTERP)); + ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) && + isResultRowInterpolated(pr, RESULT_ROW_END_INTERP)); tdListPopHead(pResultRowInfo->openWindow); continue; } STimeWindow w = pr->win; - int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pInfo->binfo.pCtx, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t ret = + setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - SGroupKeys *pTsKey = taosArrayGet(pInfo->pPrevValues, 0); - int64_t prevTs = *(int64_t*) pTsKey->pData; - doTimeWindowInterpolation(pInfo, numOfOutput, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, - w.ekey, RESULT_ROW_END_INTERP); + SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0); + int64_t prevTs = *(int64_t*)pTsKey->pData; + doTimeWindowInterpolation(pInfo, numOfOutput, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, w.ekey, + RESULT_ROW_END_INTERP); setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); setNotInterpoWindowKey(pInfo->binfo.pCtx, numOfExprs, RESULT_ROW_START_INTERP); @@ -670,7 +669,7 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) { closeResultRow(pr); tdListPopHead(pResultRowInfo->openWindow); - } else { // the remains are can not be closed yet. + } else { // the remains are can not be closed yet. break; } } @@ -678,10 +677,9 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num typedef int64_t (*__get_value_fn_t)(void* data, int32_t index); -int32_t binarySearch(void* keyList, int num, TSKEY key, int order, - __get_value_fn_t getValuefn) { - int firstPos = 0, lastPos = num - 1, midPos = -1; - int numOfRows = 0; +int32_t binarySearch(void* keyList, int num, TSKEY key, int order, __get_value_fn_t getValuefn) { + int firstPos = 0, lastPos = num - 1, midPos = -1; + int numOfRows = 0; if (num <= 0) return -1; if (order == TSDB_ORDER_DESC) { @@ -734,7 +732,7 @@ int32_t binarySearch(void* keyList, int num, TSKEY key, int order, } int64_t getReskey(void* data, int32_t index) { - SArray* res = (SArray*) data; + SArray* res = (SArray*)data; SResKeyPos* pos = taosArrayGetP(res, index); return *(int64_t*)pos->key; } @@ -760,7 +758,7 @@ static int32_t saveResult(SResultRow* result, uint64_t groupId, SArray* pUpdated newPos->groupId = groupId; newPos->pos = (SResultRowPosition){.pageId = result->pageId, .offset = result->offset}; *(int64_t*)newPos->key = result->win.skey; - if (taosArrayInsert(pUpdated, index, &newPos) == NULL ){ + if (taosArrayInsert(pUpdated, index, &newPos) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } return TSDB_CODE_SUCCESS; @@ -772,19 +770,20 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - int32_t startPos = 0; - int32_t numOfOutput = pOperatorInfo->numOfExprs; - int64_t *tsCols = extractTsCol(pBlock, pInfo); - uint64_t tableGroupId = pBlock->info.groupId; - bool ascScan = (pInfo->order == TSDB_ORDER_ASC); - TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); - SResultRow* pResult = NULL; + int32_t startPos = 0; + int32_t numOfOutput = pOperatorInfo->numOfExprs; + int64_t* tsCols = extractTsCol(pBlock, pInfo); + uint64_t tableGroupId = pBlock->info.groupId; + bool ascScan = (pInfo->order == TSDB_ORDER_ASC); + TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); + SResultRow* pResult = NULL; STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->interval.precision, &pInfo->win); - int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t ret = + setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -799,8 +798,9 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } } - TSKEY ekey = ascScan? win.ekey:win.skey; - int32_t forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); + TSKEY ekey = ascScan ? win.ekey : win.skey; + int32_t forwardRows = + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); ASSERT(forwardRows > 0); // prev time window not interpolation yet. @@ -809,8 +809,9 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos); // restore current time window - ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + ret = + setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -834,14 +835,13 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } // null data, failed to allocate more memory buffer - int32_t code = - setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, + pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, + &pInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE_SMA) { @@ -852,16 +852,17 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } } - ekey = ascScan? nextWin.ekey:nextWin.skey; + ekey = ascScan ? nextWin.ekey : nextWin.skey; forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); // window start(end) key interpolation - doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardRows); + doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, + forwardRows); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, - pBlock->info.rows, numOfOutput, pInfo->order); + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, + tsCols, pBlock->info.rows, numOfOutput, pInfo->order); doCloseWindow(pResultRowInfo, pInfo, pResult); } @@ -880,7 +881,7 @@ void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInf SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult) { SResultRowPosition pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; - SListNode* pn = tdListGetTail(pResultRowInfo->openWindow); + SListNode* pn = tdListGetTail(pResultRowInfo->openWindow); if (pn == NULL) { tdListAppend(pResultRowInfo->openWindow, &pos); return pos; @@ -918,7 +919,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { int32_t scanFlag = MAIN_SCAN; - int64_t st = taosGetTimestampUs(); + int64_t st = taosGetTimestampUs(); SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { @@ -990,7 +991,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI struct SColumnDataAgg* pAgg = NULL; for (int32_t j = 0; j < pBlock->info.rows; ++j) { - pAgg = (pBlock->pBlockAgg != NULL)? pBlock->pBlockAgg[pInfo->stateCol.slotId]: NULL; + pAgg = (pBlock->pBlockAgg != NULL) ? pBlock->pBlockAgg[pInfo->stateCol.slotId] : NULL; if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pAgg)) { continue; } @@ -1095,7 +1096,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { doStateWindowAggImpl(pOperator, pInfo, pBlock); } - pOperator->cost.openCost = (taosGetTimestampUs() - st)/1000.0; + pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); @@ -1110,7 +1111,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { size_t rows = pBInfo->pRes->info.rows; pOperator->resultInfo.totalRows += rows; - return (rows == 0)? NULL : pBInfo->pRes; + return (rows == 0) ? NULL : pBInfo->pRes; } static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { @@ -1141,7 +1142,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { size_t rows = pBlock->info.rows; pOperator->resultInfo.totalRows += rows; - return (rows == 0)? NULL:pBlock; + return (rows == 0) ? NULL : pBlock; } } @@ -1177,7 +1178,7 @@ static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type } void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SOptrBasicInfo* pBinfo, int32_t numOfOutput) { - SResultRow* pResult = getResultRowByPos(pResultBuf, p1); + SResultRow* pResult = getResultRowByPos(pResultBuf, p1); SqlFunctionCtx* pCtx = pBinfo->pCtx; for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultCell(pResult, i, pBinfo->rowCellInfoOffset); @@ -1192,28 +1193,24 @@ void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SOptrB } } -void doClearWindow(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, char* pData, - int16_t bytes, uint64_t groupId, int32_t numOfOutput) { +void doClearWindow(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, char* pData, int16_t bytes, uint64_t groupId, + int32_t numOfOutput) { SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId); SResultRowPosition* p1 = - (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, - GET_RES_WINDOW_KEY_LEN(bytes)); + (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); doClearWindowImpl(p1, pSup->pResultBuf, pBinfo, numOfOutput); } -static void doClearWindows(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, - SInterval* pInterval, int32_t tsIndex, int32_t numOfOutput, SSDataBlock* pBlock, - SArray* pUpWins) { +static void doClearWindows(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, SInterval* pInterval, int32_t tsIndex, + int32_t numOfOutput, SSDataBlock* pBlock, SArray* pUpWins) { SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex); - TSKEY *tsCols = (TSKEY*)pColDataInfo->pData; - int32_t step = 0; + TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; + int32_t step = 0; for (int32_t i = 0; i < pBlock->info.rows; i += step) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, - pInterval->precision, NULL); - step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, - win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, pInterval->precision, NULL); + step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); doClearWindow(pSup, pBinfo, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput); if (pUpWins) { taosArrayPush(pUpWins, &win); @@ -1221,19 +1218,18 @@ static void doClearWindows(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, } } -static int32_t closeIntervalWindow(SHashObj *pHashMap, STimeWindowAggSupp *pSup, - SInterval* pInterval, SArray* closeWins) { - void *pIte = NULL; +static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, + SArray* closeWins) { + void* pIte = NULL; size_t keyLen = 0; - while((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { - void* key = taosHashGetKey(pIte, &keyLen); - uint64_t groupId = *(uint64_t*) key; + while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { + void* key = taosHashGetKey(pIte, &keyLen); + uint64_t groupId = *(uint64_t*)key; ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))); - TSKEY ts = *(int64_t*) ((char*)key + sizeof(uint64_t)); + TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, - pInterval->precision, NULL); + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, NULL); if (win.ekey < pSup->maxTs - pSup->waterMark) { if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE_SMA) { if (taosHashGet(pSup->winMap, &win.skey, sizeof(TSKEY))) { @@ -1242,8 +1238,7 @@ static int32_t closeIntervalWindow(SHashObj *pHashMap, STimeWindowAggSupp *pSup, } char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))]; SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId); - if (pSup->calTrigger != STREAM_TRIGGER_AT_ONCE_SMA && - pSup->calTrigger != STREAM_TRIGGER_WINDOW_CLOSE_SMA) { + if (pSup->calTrigger != STREAM_TRIGGER_AT_ONCE_SMA && pSup->calTrigger != STREAM_TRIGGER_WINDOW_CLOSE_SMA) { taosHashRemove(pHashMap, keyBuf, keyLen); } SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); @@ -1251,7 +1246,7 @@ static int32_t closeIntervalWindow(SHashObj *pHashMap, STimeWindowAggSupp *pSup, return TSDB_CODE_OUT_OF_MEMORY; } pos->groupId = groupId; - pos->pos = *(SResultRowPosition*) pIte; + pos->pos = *(SResultRowPosition*)pIte; *(int64_t*)pos->key = ts; if (!taosArrayPush(closeWins, &pos)) { taosMemoryFree(pos); @@ -1265,7 +1260,7 @@ static int32_t closeIntervalWindow(SHashObj *pHashMap, STimeWindowAggSupp *pSup, static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SIntervalAggOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; pInfo->order = TSDB_ORDER_ASC; @@ -1301,8 +1296,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } if (pBlock->info.type == STREAM_REPROCESS) { - doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, 0, - pOperator->numOfExprs, pBlock, NULL); + doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, 0, pOperator->numOfExprs, pBlock, NULL); qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); continue; } @@ -1311,10 +1305,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated); } - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, - &pInfo->interval, pClosed); - finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pClosed, - pInfo->binfo.rowCellInfoOffset); + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pClosed); + finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pClosed, pInfo->binfo.rowCellInfoOffset); if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE || pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE_SMA) { taosArrayAddAll(pUpdated, pClosed); @@ -1345,7 +1337,7 @@ void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { } void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { - SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo *)param; + SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); cleanupAggSup(&pInfo->aggSup); if (pInfo->pChildren) { @@ -1374,7 +1366,7 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn)); pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys)); - { // ts column + { // ts column SColumn c = {0}; c.colId = 1; c.slotId = pInfo->primaryTsIndex; @@ -1383,14 +1375,14 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt taosArrayPush(pInfo->pInterpCols, &c); SGroupKeys key = {0}; - key.bytes = c.bytes; - key.type = c.type; - key.isNull = true; // to denote no value is assigned yet - key.pData = taosMemoryCalloc(1, c.bytes); + key.bytes = c.bytes; + key.type = c.type; + key.isNull = true; // to denote no value is assigned yet + key.pData = taosMemoryCalloc(1, c.bytes); taosArrayPush(pInfo->pPrevValues, &key); } - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExpr = pCtx[i].pExpr; if (strcmp(pExpr->pExpr->_function.functionName, "twa") == 0) { @@ -1401,10 +1393,10 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt needed = true; SGroupKeys key = {0}; - key.bytes = c.bytes; - key.type = c.type; + key.bytes = c.bytes; + key.type = c.type; key.isNull = false; - key.pData = taosMemoryCalloc(1, c.bytes); + key.pData = taosMemoryCalloc(1, c.bytes); taosArrayPush(pInfo->pPrevValues, &key); } } @@ -1421,11 +1413,11 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* goto _error; } - pInfo->win = pTaskInfo->window; - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; + pInfo->win = pTaskInfo->window; + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; pInfo->execModel = pTaskInfo->execModel; - pInfo->twAggSup = *pTwAggSupp; + pInfo->twAggSup = *pTwAggSupp; pInfo->primaryTsIndex = primaryTsSlotId; @@ -1438,7 +1430,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); pInfo->invertible = allInvertible(pInfo->binfo.pCtx, numOfCols); - pInfo->invertible = false; // Todo(liuyao): Dependent TSDB API + pInfo->invertible = false; // Todo(liuyao): Dependent TSDB API pInfo->timeWindowInterpo = timeWindowinterpNeeded(pInfo->binfo.pCtx, numOfCols, pInfo); if (pInfo->timeWindowInterpo) { @@ -1452,14 +1444,14 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); - pOperator->name = "TimeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->numOfExprs = numOfCols; - pOperator->info = pInfo; + pOperator->name = "TimeIntervalAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->numOfExprs = numOfCols; + pOperator->info = pInfo; pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL, destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); @@ -1480,10 +1472,11 @@ _error: } SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { + SSDataBlock* pResBlock, SInterval* pInterval, + int32_t primaryTsSlotId, STimeWindowAggSupp* pTwAggSupp, + SExecTaskInfo* pTaskInfo) { SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -1494,19 +1487,18 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(pOperator, 4096); int32_t code = - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, - keyBufSize, pTaskInfo->id.str); + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); if (code != TSDB_CODE_SUCCESS) { goto _error; } initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); - int32_t numOfChild = 8;// Todo(liuyao) get it from phy plan + int32_t numOfChild = 8; // Todo(liuyao) get it from phy plan pInfo->pChildren = taosArrayInit(numOfChild, sizeof(SOperatorInfo)); for (int32_t i = 0; i < numOfChild; i++) { - SSDataBlock* chRes = createOneDataBlock(pResBlock, false); - SOperatorInfo* pChildOp = createIntervalOperatorInfo(NULL, pExprInfo, numOfCols, - chRes, pInterval, primaryTsSlotId, pTwAggSupp, pTaskInfo); + SSDataBlock* chRes = createOneDataBlock(pResBlock, false); + SOperatorInfo* pChildOp = createIntervalOperatorInfo(NULL, pExprInfo, numOfCols, chRes, pInterval, primaryTsSlotId, + pTwAggSupp, pTaskInfo); if (pChildOp && chRes) { taosArrayPush(pInfo->pChildren, &pChildOp); continue; @@ -1523,9 +1515,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->numOfExprs = numOfCols; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, - destroyStreamFinalIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, - NULL); + pOperator->fpSet = + createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -1573,7 +1565,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExpr initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); pOperator->name = "StreamTimeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL; pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->pExpr = pExprInfo; @@ -1722,7 +1714,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { size_t rows = pBInfo->pRes->info.rows; pOperator->resultInfo.totalRows += rows; - return (rows == 0)? NULL : pBInfo->pRes; + return (rows == 0) ? NULL : pBInfo->pRes; } static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator) { @@ -1828,15 +1820,15 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf pInfo->twAggSup = *pTwAggSup; initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - pInfo->tsSlotId = tsSlotId; - pOperator->name = "StateWindowOperator"; + pInfo->tsSlotId = tsSlotId; + pOperator->name = "StateWindowOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExpr; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExpr; pOperator->numOfExprs = numOfCols; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL, destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); @@ -1907,18 +1899,17 @@ _error: return NULL; } -static SArray* doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, - int32_t tableGroupId) { +static SArray* doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; - SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); - SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - int32_t numOfOutput = pOperatorInfo->numOfExprs; - SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); - int32_t step = 1; - bool ascScan = true; - TSKEY* tsCols = NULL; - SResultRow* pResult = NULL; - int32_t forwardRows = 0; + SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + int32_t numOfOutput = pOperatorInfo->numOfExprs; + SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); + int32_t step = 1; + bool ascScan = true; + TSKEY* tsCols = NULL; + SResultRow* pResult = NULL; + int32_t forwardRows = 0; if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); @@ -1927,14 +1918,13 @@ static SArray* doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataB return pUpdated; } - int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); - TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); - STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, - &pInfo->interval, pInfo->interval.precision, NULL); + int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); + TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); + STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, + pInfo->interval.precision, NULL); while (1) { - int32_t code = - setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pInfo->binfo.pCtx, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1943,14 +1933,15 @@ static SArray* doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataB pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; *(int64_t*)pos->key = pResult->win.skey; taosArrayPush(pUpdated, &pos); - forwardRows = - getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, + TSDB_ORDER_ASC); // window start(end) key interpolation // disable it temporarily -// doWindowBorderInterpolation(pInfo, pSDataBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardRows); + // doWindowBorderInterpolation(pInfo, pSDataBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, + // forwardRows); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, - pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, + tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); int32_t prevEndPos = (forwardRows - 1) * step + startPos; startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order); if (startPos < 0) { @@ -1960,12 +1951,10 @@ static SArray* doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataB return pUpdated; } -bool isFinalInterval(SStreamFinalIntervalOperatorInfo* pInfo) { - return pInfo->pChildren != NULL; -} +bool isFinalInterval(SStreamFinalIntervalOperatorInfo* pInfo) { return pInfo->pChildren != NULL; } -void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, - int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { +void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput, + SExecTaskInfo* pTaskInfo) { for (int32_t k = 0; k < numOfOutput; ++k) { if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) { continue; @@ -1982,24 +1971,22 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, } } -static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SArray *pWinArray, - int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { +static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SArray* pWinArray, int32_t groupId, + int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { int32_t size = taosArrayGetSize(pWinArray); ASSERT(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { STimeWindow* pParentWin = taosArrayGet(pWinArray, i); - SResultRow* pCurResult = NULL; - setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, pParentWin, true, &pCurResult, 0, - pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, - pTaskInfo); + SResultRow* pCurResult = NULL; + setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, pParentWin, true, &pCurResult, 0, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); for (int32_t j = 0; j < numOfChildren; j++) { - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, j); + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, j); SIntervalAggOperatorInfo* pChInfo = pChildOp->info; - SResultRow* pChResult = NULL; - setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, pParentWin, true, &pChResult, - 0, pChInfo->binfo.pCtx, pChildOp->numOfExprs, pChInfo->binfo.rowCellInfoOffset, - &pChInfo->aggSup, pTaskInfo); + SResultRow* pChResult = NULL; + setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, pParentWin, true, &pChResult, 0, pChInfo->binfo.pCtx, + pChildOp->numOfExprs, pChInfo->binfo.rowCellInfoOffset, &pChInfo->aggSup, pTaskInfo); compactFunctions(pInfo->binfo.pCtx, pChInfo->binfo.pCtx, numOfOutput, pTaskInfo); } } @@ -2007,8 +1994,8 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SArra static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = NULL; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + SArray* pUpdated = NULL; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -2028,23 +2015,23 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, MAIN_SCAN, true); if (pBlock->info.type == STREAM_REPROCESS) { - SArray *pUpWins = taosArrayInit(8, sizeof(STimeWindow)); - doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, - pInfo->primaryTsIndex, pOperator->numOfExprs, pBlock, pUpWins); + SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); + doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, pInfo->primaryTsIndex, pOperator->numOfExprs, + pBlock, pUpWins); if (isFinalInterval(pInfo)) { - int32_t childIndex = 0; //Todo(liuyao) get child id from SSDataBlock - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); + int32_t childIndex = 0; // Todo(liuyao) get child id from SSDataBlock + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SIntervalAggOperatorInfo* pChildInfo = pChildOp->info; - doClearWindows(&pChildInfo->aggSup, &pChildInfo->binfo, &pChildInfo->interval, - pChildInfo->primaryTsIndex, pChildOp->numOfExprs, pBlock, NULL); - rebuildIntervalWindow(pInfo, pUpWins, pInfo->binfo.pRes->info.groupId, - pOperator->numOfExprs, pOperator->pTaskInfo); + doClearWindows(&pChildInfo->aggSup, &pChildInfo->binfo, &pChildInfo->interval, pChildInfo->primaryTsIndex, + pChildOp->numOfExprs, pBlock, NULL); + rebuildIntervalWindow(pInfo, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->numOfExprs, + pOperator->pTaskInfo); } taosArrayDestroy(pUpWins); continue; } if (isFinalInterval(pInfo)) { - int32_t chIndex = 1; //Todo(liuyao) get it from SSDataBlock + int32_t chIndex = 1; // Todo(liuyao) get it from SSDataBlock SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); doStreamIntervalAgg(pChildOp); } @@ -2073,7 +2060,7 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) { if (pInfo->pChildren != NULL) { int32_t size = taosArrayGetSize(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { - SOperatorInfo *pChild = taosArrayGetP(pInfo->pChildren, i); + SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i); SStreamSessionAggOperatorInfo* pChInfo = pChild->info; destroyStreamSessionAggOperatorInfo(pChInfo, numOfOutput); taosMemoryFreeClear(pChild); @@ -2082,8 +2069,8 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) { } } -int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo, - int32_t numOfCols, SSDataBlock* pResultBlock, SDiskbasedBuf* pResultBuf) { +int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, + SDiskbasedBuf* pResultBuf) { pBasicInfo->pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pBasicInfo->rowCellInfoOffset); pBasicInfo->pRes = pResultBlock; for (int32_t i = 0; i < numOfCols; ++i) { @@ -2097,22 +2084,20 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num pDummy[i].functionId = pCtx[i].functionId; } } -void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, - int64_t waterMark, uint8_t type) { +void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark, + uint8_t type) { ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); SStreamBlockScanInfo* pScanInfo = downstream->info; - pScanInfo->sessionSup = - (SessionWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type}; + pScanInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type}; pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark); } -SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, - SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, - int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { - int32_t code = TSDB_CODE_OUT_OF_MEMORY; - SStreamSessionAggOperatorInfo* pInfo = - taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); +SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, + STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { + int32_t code = TSDB_CODE_OUT_OF_MEMORY; + SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -2124,13 +2109,12 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, goto _error; } - code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock, - pInfo->streamAggSup.pResultBuf); + code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock, pInfo->streamAggSup.pResultBuf); if (code != TSDB_CODE_SUCCESS) { goto _error; } pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols); - + pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx)); if (pInfo->pDummyCtx == NULL) { goto _error; @@ -2159,12 +2143,11 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, pOperator->pExpr = pExprInfo; pOperator->numOfExprs = numOfCols; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, - NULL, NULL, destroyStreamSessionAggOperatorInfo, aggEncodeResultRow, - aggDecodeResultRow, NULL); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); pOperator->pTaskInfo = pTaskInfo; - initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, - pInfo->twAggSup.waterMark, pOperator->operatorType); + initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -2180,42 +2163,37 @@ _error: } int64_t getSessionWindowEndkey(void* data, int32_t index) { - SArray* pWinInfos = (SArray*) data; + SArray* pWinInfos = (SArray*)data; SResultWindowInfo* pWin = taosArrayGet(pWinInfos, index); return pWin->win.ekey; } static bool isInWindow(SResultWindowInfo* pWin, TSKEY ts, int64_t gap) { int64_t sGap = ts - pWin->win.skey; int64_t eGap = pWin->win.ekey - ts; - if ( (sGap < 0 && sGap >= -gap) || (eGap < 0 && eGap >= -gap) || (sGap >= 0 && eGap >= 0) ) { + if ((sGap < 0 && sGap >= -gap) || (eGap < 0 && eGap >= -gap) || (sGap >= 0 && eGap >= 0)) { return true; } return false; } -static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY ts, - int32_t index) { - SResultWindowInfo win = - {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false}; +static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY ts, int32_t index) { + SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false}; return taosArrayInsert(pWinInfos, index, &win); } static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY ts) { - SResultWindowInfo win = - {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false}; + SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false}; return taosArrayPush(pWinInfos, &win); } -SResultWindowInfo* getSessionTimeWindow(SArray* pWinInfos, TSKEY ts, int64_t gap, - int32_t* pIndex) { +SResultWindowInfo* getSessionTimeWindow(SArray* pWinInfos, TSKEY ts, int64_t gap, int32_t* pIndex) { int32_t size = taosArrayGetSize(pWinInfos); if (size == 0) { *pIndex = 0; return addNewSessionWindow(pWinInfos, ts); } // find the first position which is smaller than the key - int32_t index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, - getSessionWindowEndkey); + int32_t index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getSessionWindowEndkey); SResultWindowInfo* pWin = NULL; if (index >= 0) { pWin = taosArrayGet(pWinInfos, index); @@ -2241,8 +2219,8 @@ SResultWindowInfo* getSessionTimeWindow(SArray* pWinInfos, TSKEY ts, int64_t gap return insertNewSessionWindow(pWinInfos, ts, index + 1); } -int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t rows, - int32_t start, int64_t gap, SHashObj* pStDeleted) { +int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t rows, int32_t start, int64_t gap, + SHashObj* pStDeleted) { for (int32_t i = start; i < rows; ++i) { if (!isInWindow(pWinInfo, pTs[i], gap)) { return i - start; @@ -2259,16 +2237,16 @@ int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t return rows - start; } -static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, - SqlFunctionCtx* pCtx, int32_t groupId, int32_t numOfOutput, - int32_t* rowCellInfoOffset, SStreamAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) { +static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx, + int32_t groupId, int32_t numOfOutput, int32_t* rowCellInfoOffset, + SStreamAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) { assert(pWinInfo->win.skey <= pWinInfo->win.ekey); // too many time window in query int32_t size = taosArrayGetSize(pAggSup->pResultRows); if (size > MAX_INTERVAL_TIME_WINDOW) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); } - + if (pWinInfo->pos.pageId == -1) { *pResult = getNewResultRow(pAggSup->pResultBuf, groupId, pAggSup->resultRowSize); if (*pResult == NULL) { @@ -2293,51 +2271,47 @@ static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pRes return TSDB_CODE_SUCCESS; } -static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, - SStreamAggSupporter* pAggSup, SColumnInfoData* pTimeWindowData, - SSDataBlock* pSDataBlock, SResultWindowInfo* pCurWin, SResultRow** pResult, - int32_t startIndex, int32_t winRows, int32_t numOutput, SExecTaskInfo* pTaskInfo) { - SColumnInfoData* pColDataInfo = - taosArrayGet(pSDataBlock->pDataBlock, tsColId); - TSKEY* tsCols = (int64_t*)pColDataInfo->pData; - int32_t code = setWindowOutputBuf(pCurWin, pResult, pBinfo->pCtx, - pSDataBlock->info.groupId, numOutput, pBinfo->rowCellInfoOffset, - pAggSup, pTaskInfo); +static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStreamAggSupporter* pAggSup, + SColumnInfoData* pTimeWindowData, SSDataBlock* pSDataBlock, + SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows, + int32_t numOutput, SExecTaskInfo* pTaskInfo) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, tsColId); + TSKEY* tsCols = (int64_t*)pColDataInfo->pData; + int32_t code = setWindowOutputBuf(pCurWin, pResult, pBinfo->pCtx, pSDataBlock->info.groupId, numOutput, + pBinfo->rowCellInfoOffset, pAggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, true); - doApplyFunctions(pTaskInfo, pBinfo->pCtx, &pCurWin->win, pTimeWindowData, startIndex, - winRows, tsCols, pSDataBlock->info.rows, numOutput, TSDB_ORDER_ASC); + doApplyFunctions(pTaskInfo, pBinfo->pCtx, &pCurWin->win, pTimeWindowData, startIndex, winRows, tsCols, + pSDataBlock->info.rows, numOutput, TSDB_ORDER_ASC); return TSDB_CODE_SUCCESS; } -static int32_t doOneWindowAgg(SStreamSessionAggOperatorInfo* pInfo, - SSDataBlock* pSDataBlock, SResultWindowInfo* pCurWin, SResultRow** pResult, - int32_t startIndex, int32_t winRows, int32_t numOutput, SExecTaskInfo* pTaskInfo) { - return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, - &pInfo->twAggSup.timeWindowData, pSDataBlock, pCurWin, pResult, startIndex, - winRows, numOutput, pTaskInfo); +static int32_t doOneWindowAgg(SStreamSessionAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock, + SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows, + int32_t numOutput, SExecTaskInfo* pTaskInfo) { + return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData, + pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pTaskInfo); } -static int32_t doOneStateWindowAgg(SStreamStateAggOperatorInfo* pInfo, - SSDataBlock* pSDataBlock, SResultWindowInfo* pCurWin, SResultRow** pResult, - int32_t startIndex, int32_t winRows, int32_t numOutput, SExecTaskInfo* pTaskInfo) { - return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, - &pInfo->twAggSup.timeWindowData, pSDataBlock, pCurWin, pResult, startIndex, - winRows, numOutput, pTaskInfo); +static int32_t doOneStateWindowAgg(SStreamStateAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock, + SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, + int32_t winRows, int32_t numOutput, SExecTaskInfo* pTaskInfo) { + return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData, + pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pTaskInfo); } -int32_t copyWinInfoToDataBlock(SSDataBlock* pBlock, SStreamAggSupporter* pAggSup, - int32_t start, int32_t num, int32_t numOfExprs, SOptrBasicInfo* pBinfo) { +int32_t copyWinInfoToDataBlock(SSDataBlock* pBlock, SStreamAggSupporter* pAggSup, int32_t start, int32_t num, + int32_t numOfExprs, SOptrBasicInfo* pBinfo) { for (int32_t i = start; i < num; i += 1) { SResultWindowInfo* pWinInfo = taosArrayGet(pAggSup->pResultRows, start); - SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, pWinInfo->pos.pageId); - SResultRow* pRow = (SResultRow*)((char*)bufPage + pWinInfo->pos.offset); + SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, pWinInfo->pos.pageId); + SResultRow* pRow = (SResultRow*)((char*)bufPage + pWinInfo->pos.offset); for (int32_t j = 0; j < numOfExprs; ++j) { SResultRowEntryInfo* pResultInfo = getResultCell(pRow, j, pBinfo->rowCellInfoOffset); - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, j); - char* in = GET_ROWCELL_INTERBUF(pBinfo->pCtx[j].resultInfo); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, j); + char* in = GET_ROWCELL_INTERBUF(pBinfo->pCtx[j].resultInfo); colDataAppend(pColInfoData, pBlock->info.rows, in, pResultInfo->isNullRes); } pBlock->info.rows += pRow->numOfRows; @@ -2349,7 +2323,7 @@ int32_t copyWinInfoToDataBlock(SSDataBlock* pBlock, SStreamAggSupporter* pAggSup int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) { SResultWindowInfo* pCurWin = taosArrayGet(pWinInfos, startIndex); - int32_t size = taosArrayGetSize(pWinInfos); + int32_t size = taosArrayGetSize(pWinInfos); // Just look for the window behind StartIndex for (int32_t i = startIndex + 1; i < size; i++) { SResultWindowInfo* pWinInfo = taosArrayGet(pWinInfos, i); @@ -2361,20 +2335,20 @@ int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) return size - startIndex - 1; } -void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, - int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SHashObj* pStUpdated, SHashObj* pStDeleted) { +void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, int32_t groupId, + int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SHashObj* pStUpdated, SHashObj* pStDeleted) { SResultWindowInfo* pCurWin = taosArrayGet(pInfo->streamAggSup.pResultRows, startIndex); - SResultRow* pCurResult = NULL; - setWindowOutputBuf(pCurWin, &pCurResult, pInfo->binfo.pCtx, groupId, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->streamAggSup, pTaskInfo); + SResultRow* pCurResult = NULL; + setWindowOutputBuf(pCurWin, &pCurResult, pInfo->binfo.pCtx, groupId, numOfOutput, pInfo->binfo.rowCellInfoOffset, + &pInfo->streamAggSup, pTaskInfo); num += startIndex + 1; ASSERT(num <= taosArrayGetSize(pInfo->streamAggSup.pResultRows)); // Just look for the window behind StartIndex for (int32_t i = startIndex + 1; i < num; i++) { SResultWindowInfo* pWinInfo = taosArrayGet(pInfo->streamAggSup.pResultRows, i); - SResultRow* pWinResult = NULL; - setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->streamAggSup, pTaskInfo); + SResultRow* pWinResult = NULL; + setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, numOfOutput, pInfo->binfo.rowCellInfoOffset, + &pInfo->streamAggSup, pTaskInfo); pCurWin->win.ekey = TMAX(pCurWin->win.ekey, pWinInfo->win.ekey); compactFunctions(pInfo->binfo.pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo); taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition)); @@ -2386,43 +2360,41 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, } } -static void doStreamSessionAggImpl(SOperatorInfo* pOperator, - SSDataBlock* pSDataBlock, SHashObj* pStUpdated, SHashObj* pStDeleted) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; +static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated, + SHashObj* pStDeleted) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - bool masterScan = true; - int32_t numOfOutput = pOperator->numOfExprs; - int64_t groupId = pSDataBlock->info.groupId; - int64_t gap = pInfo->gap; - int64_t code = TSDB_CODE_SUCCESS; + bool masterScan = true; + int32_t numOfOutput = pOperator->numOfExprs; + int64_t groupId = pSDataBlock->info.groupId; + int64_t gap = pInfo->gap; + int64_t code = TSDB_CODE_SUCCESS; - int32_t step = 1; - bool ascScan = true; - TSKEY* tsCols = NULL; + int32_t step = 1; + bool ascScan = true; + TSKEY* tsCols = NULL; SResultRow* pResult = NULL; - int32_t winRows = 0; + int32_t winRows = 0; if (pSDataBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = - taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; } else { - return ; + return; } - + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; - for(int32_t i = 0; i < pSDataBlock->info.rows; ) { - int32_t winIndex = 0; - SResultWindowInfo* pCurWin = - getSessionTimeWindow(pAggSup->pResultRows, tsCols[i], gap, &winIndex); - winRows = updateSessionWindowInfo(pCurWin, tsCols, pSDataBlock->info.rows, i, - pInfo->gap, pStDeleted); + for (int32_t i = 0; i < pSDataBlock->info.rows;) { + int32_t winIndex = 0; + SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup->pResultRows, tsCols[i], gap, &winIndex); + winRows = updateSessionWindowInfo(pCurWin, tsCols, pSDataBlock->info.rows, i, pInfo->gap, pStDeleted); code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } // window start(end) key interpolation - // doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardRows, + // doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, + // forwardRows, // pInfo->order, false); int32_t winNum = getNumCompactWindow(pAggSup->pResultRows, winIndex, gap); if (winNum > 0) { @@ -2440,15 +2412,14 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, } } -static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SOptrBasicInfo* pBinfo, - SSDataBlock* pBlock, int32_t tsIndex, int32_t numOfOutput, int64_t gap, SArray* result) { +static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SOptrBasicInfo* pBinfo, SSDataBlock* pBlock, + int32_t tsIndex, int32_t numOfOutput, int64_t gap, SArray* result) { SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex); - TSKEY *tsCols = (TSKEY*)pColDataInfo->pData; - int32_t step = 0; + TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; + int32_t step = 0; for (int32_t i = 0; i < pBlock->info.rows; i += step) { - int32_t winIndex = 0; - SResultWindowInfo* pCurWin = - getSessionTimeWindow(pAggSup->pResultRows, tsCols[i], gap, &winIndex); + int32_t winIndex = 0; + SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup->pResultRows, tsCols[i], gap, &winIndex); step = updateSessionWindowInfo(pCurWin, tsCols, pBlock->info.rows, i, gap, NULL); ASSERT(isInWindow(pCurWin, tsCols[i], gap)); doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pBinfo, numOfOutput); @@ -2459,9 +2430,9 @@ static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SOptrBasicInfo* } static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated, int32_t groupId) { - void* pData = NULL; + void* pData = NULL; size_t keyLen = 0; - while((pData = taosHashIterate(pStUpdated, pData)) != NULL) { + while ((pData = taosHashIterate(pStUpdated, pData)) != NULL) { void* key = taosHashGetKey(pData, &keyLen); ASSERT(keyLen == sizeof(SResultRowPosition)); SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); @@ -2479,12 +2450,12 @@ static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated, int32_t void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) { blockDataCleanup(pBlock); size_t keyLen = 0; - while(( (*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) { + while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false); + colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false); for (int32_t i = 1; i < pBlock->info.numOfCols; i++) { pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - colDataAppendNULL(pColInfoData, pBlock->info.rows); + colDataAppendNULL(pColInfoData, pBlock->info.rows); } pBlock->info.rows += 1; if (pBlock->info.rows + 1 >= pBlock->info.capacity) { @@ -2496,29 +2467,28 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It } } -static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray *pWinArray, - int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { +static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t groupId, + int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { int32_t size = taosArrayGetSize(pWinArray); ASSERT(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i); - SResultRow* pCurResult = NULL; - setWindowOutputBuf(pParentWin, &pCurResult, pInfo->binfo.pCtx, groupId, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->streamAggSup, pTaskInfo); + SResultRow* pCurResult = NULL; + setWindowOutputBuf(pParentWin, &pCurResult, pInfo->binfo.pCtx, groupId, numOfOutput, pInfo->binfo.rowCellInfoOffset, + &pInfo->streamAggSup, pTaskInfo); int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); for (int32_t j = 0; j < numOfChildren; j++) { - SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, j); + SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, j); SStreamSessionAggOperatorInfo* pChInfo = pChild->info; - SArray* pChWins = pChInfo->streamAggSup.pResultRows; - int32_t chWinSize = taosArrayGetSize(pChWins); - int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, - TSDB_ORDER_DESC, getSessionWindowEndkey); + SArray* pChWins = pChInfo->streamAggSup.pResultRows; + int32_t chWinSize = taosArrayGetSize(pChWins); + int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, TSDB_ORDER_DESC, getSessionWindowEndkey); for (int32_t k = index; k > 0 && k < chWinSize; k++) { SResultWindowInfo* pcw = taosArrayGet(pChWins, k); if (pParentWin->win.skey <= pcw->win.skey && pcw->win.ekey <= pParentWin->win.ekey) { SResultRow* pChResult = NULL; - setWindowOutputBuf(pcw, &pChResult, pChInfo->binfo.pCtx, groupId, - numOfOutput, pChInfo->binfo.rowCellInfoOffset, &pChInfo->streamAggSup, pTaskInfo); + setWindowOutputBuf(pcw, &pChResult, pChInfo->binfo.pCtx, groupId, numOfOutput, + pChInfo->binfo.rowCellInfoOffset, &pChInfo->streamAggSup, pTaskInfo); compactFunctions(pInfo->binfo.pCtx, pChInfo->binfo.pCtx, numOfOutput, pTaskInfo); continue; } @@ -2528,25 +2498,19 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray *pWin } } -bool isFinalSession(SStreamSessionAggOperatorInfo* pInfo) { - return pInfo->pChildren != NULL; -} +bool isFinalSession(SStreamSessionAggOperatorInfo* pInfo) { return pInfo->pChildren != NULL; } -typedef SResultWindowInfo* (*__get_win_info_)(void *); -SResultWindowInfo* getSessionWinInfo(void* pData) { - return (SResultWindowInfo*) pData; -} -SResultWindowInfo* getStateWinInfo(void* pData) { - return &((SStateWindowInfo*) pData)->winInfo; -} +typedef SResultWindowInfo* (*__get_win_info_)(void*); +SResultWindowInfo* getSessionWinInfo(void* pData) { return (SResultWindowInfo*)pData; } +SResultWindowInfo* getStateWinInfo(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; } -int32_t closeSessionWindow(SArray *pWins, STimeWindowAggSupp *pTwSup, SArray *pClosed, - int8_t calTrigger, __get_win_info_ fn) { +int32_t closeSessionWindow(SArray* pWins, STimeWindowAggSupp* pTwSup, SArray* pClosed, int8_t calTrigger, + __get_win_info_ fn) { // Todo(liuyao) save window to tdb - int32_t size = taosArrayGetSize(pWins); + int32_t size = taosArrayGetSize(pWins); for (int32_t i = 0; i < size; i++) { - void *pWin = taosArrayGet(pWins, i); - SResultWindowInfo *pSeWin = fn(pWin); + void* pWin = taosArrayGet(pWins, i); + SResultWindowInfo* pSeWin = fn(pWin); if (pSeWin->win.ekey < pTwSup->maxTs - pTwSup->waterMark) { if (!pSeWin->isClosed) { SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); @@ -2578,23 +2542,21 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - SOptrBasicInfo* pBInfo = &pInfo->binfo; + SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, - pInfo->streamAggSup.pResultBuf); - if (pBInfo->pRes->info.rows == 0 || - !hashRemainDataInGroupInfo(&pInfo->groupResInfo)) { + doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + if (pBInfo->pRes->info.rows == 0 || !hashRemainDataInGroupInfo(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + SHashObj* pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK); SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); @@ -2603,23 +2565,22 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - if (pBlock->info.type == STREAM_REPROCESS) { - SArray *pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); - doClearSessionWindows(&pInfo->streamAggSup, &pInfo->binfo, pBlock, 0, - pOperator->numOfExprs, pInfo->gap, pWins); + if (pBlock->info.type == STREAM_REPROCESS) { + SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); + doClearSessionWindows(&pInfo->streamAggSup, &pInfo->binfo, pBlock, 0, pOperator->numOfExprs, pInfo->gap, pWins); if (isFinalSession(pInfo)) { - int32_t childIndex = 0; //Todo(liuyao) get child id from SSDataBlock - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); + int32_t childIndex = 0; // Todo(liuyao) get child id from SSDataBlock + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; - doClearSessionWindows(&pChildInfo->streamAggSup, &pChildInfo->binfo, pBlock, 0, - pChildOp->numOfExprs, pChildInfo->gap, NULL); + doClearSessionWindows(&pChildInfo->streamAggSup, &pChildInfo->binfo, pBlock, 0, pChildOp->numOfExprs, + pChildInfo->gap, NULL); rebuildTimeWindow(pInfo, pWins, pInfo->binfo.pRes->info.groupId, pOperator->numOfExprs, pOperator->pTaskInfo); } taosArrayDestroy(pWins); continue; } if (isFinalSession(pInfo)) { - int32_t childIndex = 0; //Todo(liuyao) get child id from SSDataBlock + int32_t childIndex = 0; // Todo(liuyao) get child id from SSDataBlock SOptrBasicInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); doStreamSessionAggImpl(pOperator, pBlock, NULL, NULL); } @@ -2630,8 +2591,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { pOperator->status = OP_RES_TO_RETURN; SArray* pClosed = taosArrayInit(16, POINTER_BYTES); - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pClosed, - pInfo->twAggSup.calTrigger, getSessionWinInfo); + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pClosed, pInfo->twAggSup.calTrigger, + getSessionWinInfo); SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); copyUpdateResult(pStUpdated, pUpdated, pBInfo->pRes->info.groupId); taosHashCleanup(pStUpdated); @@ -2640,36 +2601,36 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } finalizeUpdatedResult(pOperator->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, - pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.rowCellInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, - pInfo->streamAggSup.pResultBuf); + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } -SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, - SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, - int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { - int32_t code = TSDB_CODE_OUT_OF_MEMORY; +SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, + int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, + int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, + SExecTaskInfo* pTaskInfo) { + int32_t code = TSDB_CODE_OUT_OF_MEMORY; SStreamSessionAggOperatorInfo* pInfo = NULL; - SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pExprInfo, - numOfCols, pResBlock, gap, tsSlotId, pTwAggSupp, pTaskInfo); + SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pExprInfo, numOfCols, pResBlock, gap, + tsSlotId, pTwAggSupp, pTaskInfo); if (pOperator == NULL) { goto _error; } pOperator->name = "StreamFinalSessionWindowAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION_WINDOW; - int32_t numOfChild = 1; //Todo(liuyao) get it from phy plan + int32_t numOfChild = 1; // Todo(liuyao) get it from phy plan pInfo = pOperator->info; - pInfo->pChildren = taosArrayInit(8, sizeof(void *)); + pInfo->pChildren = taosArrayInit(8, sizeof(void*)); for (int32_t i = 0; i < numOfChild; i++) { - SOperatorInfo* pChild = createStreamSessionAggOperatorInfo(NULL, pExprInfo, - numOfCols, NULL, gap, tsSlotId, pTwAggSupp, pTaskInfo); + SOperatorInfo* pChild = + createStreamSessionAggOperatorInfo(NULL, pExprInfo, numOfCols, NULL, gap, tsSlotId, pTwAggSupp, pTaskInfo); if (pChild == NULL) { goto _error; } @@ -2689,14 +2650,14 @@ _error: } void destroyStreamStateOperatorInfo(void* param, int32_t numOfOutput) { - SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo *)param; + SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); destroyStreamAggSupporter(&pInfo->streamAggSup); cleanupGroupResInfo(&pInfo->groupResInfo); if (pInfo->pChildren != NULL) { int32_t size = taosArrayGetSize(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { - SOperatorInfo *pChild = taosArrayGetP(pInfo->pChildren, i); + SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i); SStreamSessionAggOperatorInfo* pChInfo = pChild->info; destroyStreamSessionAggOperatorInfo(pChInfo, numOfOutput); taosMemoryFreeClear(pChild); @@ -2710,16 +2671,18 @@ int64_t getStateWinTsKey(void* data, int32_t index) { return pStateWin->winInfo.win.ekey; } -SStateWindowInfo* addNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, - SColumn* pCol) { - SStateWindowInfo win = {.stateKey.bytes = pCol->bytes, - .stateKey.type = pCol->type, - .stateKey.pData = taosMemoryCalloc(1, pCol->bytes), - .winInfo.pos.offset = -1, .winInfo.pos.pageId = -1, - .winInfo.win.skey = ts, .winInfo.win.ekey = ts, - .winInfo.isOutput = false, - .winInfo.isClosed = false, - }; +SStateWindowInfo* addNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, SColumn* pCol) { + SStateWindowInfo win = { + .stateKey.bytes = pCol->bytes, + .stateKey.type = pCol->type, + .stateKey.pData = taosMemoryCalloc(1, pCol->bytes), + .winInfo.pos.offset = -1, + .winInfo.pos.pageId = -1, + .winInfo.win.skey = ts, + .winInfo.win.ekey = ts, + .winInfo.isOutput = false, + .winInfo.isClosed = false, + }; if (IS_VAR_DATA_TYPE(win.stateKey.type)) { varDataCopy(win.stateKey.pData, pKeyData); } else { @@ -2728,16 +2691,18 @@ SStateWindowInfo* addNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, return taosArrayPush(pWinInfos, &win); } -SStateWindowInfo* insertNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, - int32_t index, SColumn* pCol) { - SStateWindowInfo win = {.stateKey.bytes = pCol->bytes, - .stateKey.type = pCol->type, - .stateKey.pData = taosMemoryCalloc(1, pCol->bytes), - .winInfo.pos.offset = -1, .winInfo.pos.pageId = -1, - .winInfo.win.skey = ts, .winInfo.win.ekey = ts, - .winInfo.isOutput = false, - .winInfo.isClosed = false, - }; +SStateWindowInfo* insertNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, int32_t index, SColumn* pCol) { + SStateWindowInfo win = { + .stateKey.bytes = pCol->bytes, + .stateKey.type = pCol->type, + .stateKey.pData = taosMemoryCalloc(1, pCol->bytes), + .winInfo.pos.offset = -1, + .winInfo.pos.pageId = -1, + .winInfo.win.skey = ts, + .winInfo.win.ekey = ts, + .winInfo.isOutput = false, + .winInfo.isClosed = false, + }; if (IS_VAR_DATA_TYPE(win.stateKey.type)) { varDataCopy(win.stateKey.pData, pKeyData); } else { @@ -2758,8 +2723,8 @@ bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) { } SStateWindowInfo* getStateWindowByTs(SArray* pWinInfos, TSKEY ts, int32_t* pIndex) { - int32_t size = taosArrayGetSize(pWinInfos); - int32_t index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey); + int32_t size = taosArrayGetSize(pWinInfos); + int32_t index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey); SStateWindowInfo* pWin = NULL; if (index >= 0) { pWin = taosArrayGet(pWinInfos, index); @@ -2780,14 +2745,13 @@ SStateWindowInfo* getStateWindowByTs(SArray* pWinInfos, TSKEY ts, int32_t* pInde return NULL; } -SStateWindowInfo* getStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, - SColumn* pCol, int32_t* pIndex) { +SStateWindowInfo* getStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, SColumn* pCol, int32_t* pIndex) { int32_t size = taosArrayGetSize(pWinInfos); if (size == 0) { *pIndex = 0; return addNewStateWindow(pWinInfos, ts, pKeyData, pCol); } - int32_t index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey); + int32_t index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey); SStateWindowInfo* pWin = NULL; if (index >= 0) { pWin = taosArrayGet(pWinInfos, index); @@ -2821,15 +2785,14 @@ SStateWindowInfo* getStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, return insertNewStateWindow(pWinInfos, ts, pKeyData, index + 1, pCol); } -int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, - SColumnInfoData* pKeyCol, int32_t rows, int32_t start, bool* allEqual, - SHashObj* pSeDelete) { +int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, SColumnInfoData* pKeyCol, int32_t rows, + int32_t start, bool* allEqual, SHashObj* pSeDelete) { *allEqual = true; SStateWindowInfo* pWinInfo = taosArrayGet(pWinInfos, winIndex); for (int32_t i = start; i < rows; ++i) { char* pKeyData = colDataGetData(pKeyCol, i); if (!isTsInWindow(pWinInfo, pTs[i])) { - if(isEqualStateKey(pWinInfo, pKeyData)) { + if (isEqualStateKey(pWinInfo, pKeyData)) { int32_t size = taosArrayGetSize(pWinInfos); if (winIndex + 1 < size) { SStateWindowInfo* pNextWin = taosArrayGet(pWinInfos, winIndex + 1); @@ -2844,7 +2807,8 @@ int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, } if (pWinInfo->winInfo.win.skey > pTs[i]) { if (pSeDelete && pWinInfo->winInfo.isOutput) { - taosHashPut(pSeDelete, &pWinInfo->winInfo.pos, sizeof(SResultRowPosition), &pWinInfo->winInfo.win.skey, sizeof(TSKEY)); + taosHashPut(pSeDelete, &pWinInfo->winInfo.pos, sizeof(SResultRowPosition), &pWinInfo->winInfo.win.skey, + sizeof(TSKEY)); pWinInfo->winInfo.isOutput = false; } pWinInfo->winInfo.win.skey = pTs[i]; @@ -2858,26 +2822,26 @@ int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, } void deleteWindow(SArray* pWinInfos, int32_t index) { - ASSERT(index >=0 && index < taosArrayGetSize(pWinInfos)); + ASSERT(index >= 0 && index < taosArrayGetSize(pWinInfos)); taosArrayRemove(pWinInfos, index); } -static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, - int32_t tsIndex, SColumn* pCol, int32_t keyIndex, SHashObj* pSeUpdated, SHashObj* pSeDeleted) { +static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int32_t tsIndex, SColumn* pCol, + int32_t keyIndex, SHashObj* pSeUpdated, SHashObj* pSeDeleted) { SColumnInfoData* pTsColInfo = taosArrayGet(pBlock->pDataBlock, tsIndex); SColumnInfoData* pKeyColInfo = taosArrayGet(pBlock->pDataBlock, keyIndex); - TSKEY* tsCol = (TSKEY *)pTsColInfo->pData; - bool allEqual = false; - int32_t step = 1; + TSKEY* tsCol = (TSKEY*)pTsColInfo->pData; + bool allEqual = false; + int32_t step = 1; for (int32_t i = 0; i < pBlock->info.rows; i += step) { - char* pKeyData = colDataGetData(pKeyColInfo, i); - int32_t winIndex = 0; + char* pKeyData = colDataGetData(pKeyColInfo, i); + int32_t winIndex = 0; SStateWindowInfo* pCurWin = getStateWindowByTs(pAggSup->pResultRows, tsCol[i], &winIndex); if (!pCurWin) { continue; } - step = updateStateWindowInfo(pAggSup->pResultRows, winIndex, tsCol, pKeyColInfo, - pBlock->info.rows, i, &allEqual, pSeDeleted); + step = updateStateWindowInfo(pAggSup->pResultRows, winIndex, tsCol, pKeyColInfo, pBlock->info.rows, i, &allEqual, + pSeDeleted); ASSERT(isTsInWindow(pCurWin, tsCol[i]) || isEqualStateKey(pCurWin, pKeyData)); taosArrayPush(pAggSup->pScanWindow, &pCurWin->winInfo.win); taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); @@ -2885,51 +2849,49 @@ static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBloc } } -static void doStreamStateAggImpl(SOperatorInfo* pOperator, - SSDataBlock* pSDataBlock, SHashObj* pSeUpdated, SHashObj* pStDeleted) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; +static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pSeUpdated, + SHashObj* pStDeleted) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamStateAggOperatorInfo* pInfo = pOperator->info; - bool masterScan = true; - int32_t numOfOutput = pOperator->numOfExprs; - int64_t groupId = pSDataBlock->info.groupId; - int64_t code = TSDB_CODE_SUCCESS; - int32_t step = 1; - bool ascScan = true; - TSKEY* tsCols = NULL; - SResultRow* pResult = NULL; - int32_t winRows = 0; + bool masterScan = true; + int32_t numOfOutput = pOperator->numOfExprs; + int64_t groupId = pSDataBlock->info.groupId; + int64_t code = TSDB_CODE_SUCCESS; + int32_t step = 1; + bool ascScan = true; + TSKEY* tsCols = NULL; + SResultRow* pResult = NULL; + int32_t winRows = 0; if (pSDataBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = - taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - tsCols = (int64_t *)pColDataInfo->pData; + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; } else { - return ; + return; } - + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; - SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); - for(int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) { - char* pKeyData = colDataGetData(pKeyColInfo, i); - int32_t winIndex = 0; - bool allEqual = true; - SStateWindowInfo* pCurWin = - getStateWindow(pAggSup->pResultRows, tsCols[i], pKeyData, &pInfo->stateCol, &winIndex); - winRows = updateStateWindowInfo(pAggSup->pResultRows, winIndex, tsCols, pKeyColInfo, - pSDataBlock->info.rows, i, &allEqual, pInfo->pSeDeleted); + SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); + for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) { + char* pKeyData = colDataGetData(pKeyColInfo, i); + int32_t winIndex = 0; + bool allEqual = true; + SStateWindowInfo* pCurWin = getStateWindow(pAggSup->pResultRows, tsCols[i], pKeyData, &pInfo->stateCol, &winIndex); + winRows = updateStateWindowInfo(pAggSup->pResultRows, winIndex, tsCols, pKeyColInfo, pSDataBlock->info.rows, i, + &allEqual, pInfo->pSeDeleted); if (!allEqual) { taosArrayPush(pAggSup->pScanWindow, &pCurWin->winInfo.win); taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); deleteWindow(pAggSup->pResultRows, winIndex); continue; } - code = doOneStateWindowAgg(pInfo, pSDataBlock, &pCurWin->winInfo, &pResult, i, winRows, - numOfOutput, pTaskInfo); + code = doOneStateWindowAgg(pInfo, pSDataBlock, &pCurWin->winInfo, &pResult, i, winRows, numOfOutput, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } pCurWin->winInfo.isClosed = false; if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition), &(pCurWin->winInfo.win.skey), sizeof(TSKEY)); + code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition), &(pCurWin->winInfo.win.skey), + sizeof(TSKEY)); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -2944,23 +2906,21 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { } SStreamStateAggOperatorInfo* pInfo = pOperator->info; - SOptrBasicInfo* pBInfo = &pInfo->binfo; + SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, - pInfo->streamAggSup.pResultBuf); - if (pBInfo->pRes->info.rows == 0 || - !hashRemainDataInGroupInfo(&pInfo->groupResInfo)) { + doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + if (pBInfo->pRes->info.rows == 0 || !hashRemainDataInGroupInfo(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pSeUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + SHashObj* pSeUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK); SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); @@ -2970,8 +2930,8 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); if (pBlock->info.type == STREAM_REPROCESS) { - doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, - &pInfo->stateCol, pInfo->stateCol.slotId, pSeUpdated, pInfo->pSeDeleted); + doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId, + pSeUpdated, pInfo->pSeDeleted); continue; } doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted); @@ -2979,10 +2939,10 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { } // restore the value pOperator->status = OP_RES_TO_RETURN; - + SArray* pClosed = taosArrayInit(16, POINTER_BYTES); - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pClosed, - pInfo->twAggSup.calTrigger, getStateWinInfo); + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pClosed, pInfo->twAggSup.calTrigger, + getStateWinInfo); SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); copyUpdateResult(pSeUpdated, pUpdated, pBInfo->pRes->info.groupId); taosHashCleanup(pSeUpdated); @@ -2991,50 +2951,49 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { } finalizeUpdatedResult(pOperator->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, - pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.rowCellInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, - pInfo->streamAggSup.pResultBuf); + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } -SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, - SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) { - SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode *)pPhyNode; - SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - int32_t tsSlotId = ((SColumnNode *)pStateNode->window.pTspk)->slotId; - SColumnNode* pColNode = (SColumnNode *)((STargetNode *)pStateNode->pStateKey)->pExpr; +SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo) { + SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode; + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; + SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; - SStreamStateAggOperatorInfo* pInfo = - taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - int32_t numOfCols = 0; + int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols); pInfo->stateCol = extractColumnFromColumnNode(pColNode); initResultSizeInfo(pOperator, 4096); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - pInfo->twAggSup = (STimeWindowAggSupp) {.waterMark = pStateNode->window.watermark, + pInfo->twAggSup = (STimeWindowAggSupp){ + .waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType, .maxTs = INT64_MIN, - .winMap = NULL,}; + .winMap = NULL, + }; initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); int32_t code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo"); if (code != TSDB_CODE_SUCCESS) { goto _error; } - code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock, - pInfo->streamAggSup.pResultBuf); + code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock, pInfo->streamAggSup.pResultBuf); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -3063,10 +3022,9 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, pOperator->pExpr = pExprInfo; pOperator->pTaskInfo = pTaskInfo; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, - NULL, destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); - initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, - pOperator->operatorType); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL, + destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 1fce5ecaba..f277d30eb1 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -316,7 +316,8 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pScanCols); CLONE_NODE_LIST_FIELD(pScanPseudoCols); - CLONE_OBJECT_FIELD(pMeta, tableMetaClone); + COPY_SCALAR_FIELD(tableType); + COPY_SCALAR_FIELD(tableId); CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone); COPY_SCALAR_FIELD(scanType); COPY_OBJECT_FIELD(scanSeq[0], sizeof(uint8_t) * 2); @@ -365,9 +366,15 @@ static SNode* logicProjectCopy(const SProjectLogicNode* pSrc, SProjectLogicNode* return (SNode*)pDst; } -static SNode* logicVnodeModifCopy(const SVnodeModifLogicNode* pSrc, SVnodeModifLogicNode* pDst) { +static SNode* logicVnodeModifCopy(const SVnodeModifyLogicNode* pSrc, SVnodeModifyLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + COPY_SCALAR_FIELD(modifyType); COPY_SCALAR_FIELD(msgType); + CLONE_NODE_FIELD(pModifyRows); + COPY_SCALAR_FIELD(tableId); + COPY_SCALAR_FIELD(tableType); + COPY_CHAR_ARRAY_FIELD(tableFName); + COPY_OBJECT_FIELD(deleteTimeRange, sizeof(STimeWindow)); return (SNode*)pDst; } @@ -400,7 +407,7 @@ static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pD COPY_SCALAR_FIELD(triggerType); COPY_SCALAR_FIELD(watermark); COPY_SCALAR_FIELD(filesFactor); - COPY_SCALAR_FIELD(stmInterAlgo); + COPY_SCALAR_FIELD(intervalAlgo); return (SNode*)pDst; } @@ -542,8 +549,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { return logicAggCopy((const SAggLogicNode*)pNode, (SAggLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_PROJECT: return logicProjectCopy((const SProjectLogicNode*)pNode, (SProjectLogicNode*)pDst); - case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: - return logicVnodeModifCopy((const SVnodeModifLogicNode*)pNode, (SVnodeModifLogicNode*)pDst); + case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: + return logicVnodeModifCopy((const SVnodeModifyLogicNode*)pNode, (SVnodeModifyLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_EXCHANGE: return logicExchangeCopy((const SExchangeLogicNode*)pNode, (SExchangeLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_MERGE: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index b5ddcff0b1..c54289bc56 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -188,8 +188,8 @@ const char* nodesNodeName(ENodeType type) { return "LogicAgg"; case QUERY_NODE_LOGIC_PLAN_PROJECT: return "LogicProject"; - case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: - return "LogicVnodeModif"; + case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: + return "LogicVnodeModify"; case QUERY_NODE_LOGIC_PLAN_EXCHANGE: return "LogicExchange"; case QUERY_NODE_LOGIC_PLAN_MERGE: @@ -228,8 +228,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiMerge"; case QUERY_NODE_PHYSICAL_PLAN_SORT: return "PhysiSort"; - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: - return "PhysiInterval"; + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: + return "PhysiHashInterval"; case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: return "PhysiStreamInterval"; case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: @@ -252,6 +252,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiDispatch"; case QUERY_NODE_PHYSICAL_PLAN_INSERT: return "PhysiInsert"; + case QUERY_NODE_PHYSICAL_PLAN_DELETE: + return "PhysiDelete"; case QUERY_NODE_PHYSICAL_SUBPLAN: return "PhysiSubplan"; case QUERY_NODE_PHYSICAL_PLAN: @@ -504,8 +506,8 @@ static int32_t jsonToLogicPlanNode(const SJson* pJson, void* pObj) { static const char* jkScanLogicPlanScanCols = "ScanCols"; static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols"; -static const char* jkScanLogicPlanTableMetaSize = "TableMetaSize"; -static const char* jkScanLogicPlanTableMeta = "TableMeta"; +static const char* jkScanLogicPlanTableId = "TableId"; +static const char* jkScanLogicPlanTableType = "TableType"; static const char* jkScanLogicPlanTagCond = "TagCond"; static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { @@ -519,10 +521,10 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { code = nodeListToJson(pJson, jkScanLogicPlanScanPseudoCols, pNode->pScanPseudoCols); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableMetaSize, TABLE_META_SIZE(pNode->pMeta)); + code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableId, pNode->tableId); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkScanLogicPlanTableMeta, tableMetaToJson, pNode->pMeta); + code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableType, pNode->tableType); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkScanLogicPlanTagCond, nodeToJson, pNode->pTagCond); @@ -543,10 +545,10 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { code = jsonToNodeList(pJson, jkScanLogicPlanScanPseudoCols, &pNode->pScanPseudoCols); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetIntValue(pJson, jkScanLogicPlanTableMetaSize, &objSize); + code = tjsonGetUBigIntValue(pJson, jkScanLogicPlanTableId, &pNode->tableId); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonMakeObject(pJson, jkScanLogicPlanTableMeta, jsonToTableMeta, (void**)&pNode->pMeta, objSize); + code = tjsonGetTinyIntValue(pJson, jkScanLogicPlanTableType, &pNode->tableType); } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkScanLogicPlanTagCond, &pNode->pTagCond); @@ -578,7 +580,7 @@ static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddIntegerToObject(pJson, jkProjectLogicPlanSlimit, pNode->slimit); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableMetaSize, pNode->soffset); + code = tjsonAddIntegerToObject(pJson, jkProjectLogicPlanSoffset, pNode->soffset); } return code; @@ -601,7 +603,45 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) { code = tjsonGetBigIntValue(pJson, jkProjectLogicPlanSlimit, &pNode->slimit); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBigIntValue(pJson, jkScanLogicPlanTableMetaSize, &pNode->soffset); + code = tjsonGetBigIntValue(pJson, jkProjectLogicPlanSoffset, &pNode->soffset); + } + + return code; +} + +static const char* jkVnodeModifyLogicPlanModifyType = "ModifyType"; +static const char* jkVnodeModifyLogicPlanMsgType = "MsgType"; +static const char* jkVnodeModifyLogicPlanModifyRows = "ModifyRows"; + +static int32_t logicVnodeModifyNodeToJson(const void* pObj, SJson* pJson) { + const SVnodeModifyLogicNode* pNode = (const SVnodeModifyLogicNode*)pObj; + + int32_t code = logicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVnodeModifyLogicPlanModifyType, pNode->modifyType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVnodeModifyLogicPlanMsgType, pNode->msgType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkVnodeModifyLogicPlanModifyRows, nodeToJson, pNode->pModifyRows); + } + + return code; +} + +static int32_t jsonToLogicVnodeModifyNode(const SJson* pJson, void* pObj) { + SVnodeModifyLogicNode* pNode = (SVnodeModifyLogicNode*)pObj; + + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + tjsonGetNumberValue(pJson, jkVnodeModifyLogicPlanModifyType, pNode->modifyType, code); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkVnodeModifyLogicPlanMsgType, &pNode->msgType); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkVnodeModifyLogicPlanModifyRows, &pNode->pModifyRows); } return code; @@ -1955,6 +1995,58 @@ static int32_t physiDispatchNodeToJson(const void* pObj, SJson* pJson) { return static int32_t jsonToPhysiDispatchNode(const SJson* pJson, void* pObj) { return jsonToPhysicDataSinkNode(pJson, pObj); } +static const char* jkDeletePhysiPlanTableId = "TableId"; +static const char* jkDeletePhysiPlanTableType = "TableType"; +static const char* jkDeletePhysiPlanTableFName = "TableFName"; +static const char* jkDeletePhysiPlanDeleteTimeRangeStartKey = "DeleteTimeRangeStartKey"; +static const char* jkDeletePhysiPlanDeleteTimeRangeEndKey = "DeleteTimeRangeEndKey"; + +static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) { + const SDataDeleterNode* pNode = (const SDataDeleterNode*)pObj; + + int32_t code = physicDataSinkNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDeletePhysiPlanTableId, pNode->tableId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDeletePhysiPlanTableType, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkDeletePhysiPlanTableFName, pNode->tableFName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDeletePhysiPlanDeleteTimeRangeStartKey, pNode->deleteTimeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDeletePhysiPlanDeleteTimeRangeEndKey, pNode->deleteTimeRange.ekey); + } + + return code; +} + +static int32_t jsonToPhysiDeleteNode(const SJson* pJson, void* pObj) { + SDataDeleterNode* pNode = (SDataDeleterNode*)pObj; + + int32_t code = jsonToPhysicDataSinkNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUBigIntValue(pJson, jkDeletePhysiPlanTableId, &pNode->tableId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkDeletePhysiPlanTableType, &pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkDeletePhysiPlanTableFName, pNode->tableFName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkDeletePhysiPlanDeleteTimeRangeStartKey, &pNode->deleteTimeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkDeletePhysiPlanDeleteTimeRangeEndKey, &pNode->deleteTimeRange.ekey); + } + + return code; +} + static const char* jkQueryNodeAddrId = "Id"; static const char* jkQueryNodeAddrInUse = "InUse"; static const char* jkQueryNodeAddrNumOfEps = "NumOfEps"; @@ -2751,7 +2843,7 @@ static const char* jkStateWindowExpr = "StateWindowExpr"; static int32_t stateWindowNodeToJson(const void* pObj, SJson* pJson) { const SStateWindowNode* pNode = (const SStateWindowNode*)pObj; - int32_t code = tjsonAddObject(pJson, jkStateWindowCol, nodeToJson, pNode->pCol); + int32_t code = tjsonAddObject(pJson, jkStateWindowCol, nodeToJson, pNode->pCol); if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkStateWindowExpr, nodeToJson, pNode->pExpr); } @@ -3522,6 +3614,73 @@ static int32_t jsonToCreateTopicStmt(const SJson* pJson, void* pObj) { return code; } +static const char* jkDeleteStmtFromTable = "FromTable"; +static const char* jkDeleteStmtWhere = "Where"; +static const char* jkDeleteStmtCountFunc = "CountFunc"; +static const char* jkDeleteStmtTagIndexCond = "TagIndexCond"; +static const char* jkDeleteStmtTimeRangeStartKey = "TimeRangeStartKey"; +static const char* jkDeleteStmtTimeRangeEndKey = "TimeRangeEndKey"; +static const char* jkDeleteStmtPrecision = "Precision"; +static const char* jkDeleteStmtDeleteZeroRows = "DeleteZeroRows"; + +static int32_t deleteStmtToJson(const void* pObj, SJson* pJson) { + const SDeleteStmt* pNode = (const SDeleteStmt*)pObj; + + int32_t code = tjsonAddObject(pJson, jkDeleteStmtFromTable, nodeToJson, pNode->pFromTable); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkDeleteStmtWhere, nodeToJson, pNode->pWhere); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkDeleteStmtCountFunc, nodeToJson, pNode->pCountFunc); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkDeleteStmtTagIndexCond, nodeToJson, pNode->pTagIndexCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDeleteStmtTimeRangeStartKey, pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDeleteStmtTimeRangeEndKey, pNode->timeRange.ekey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDeleteStmtPrecision, pNode->precision); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkDeleteStmtDeleteZeroRows, pNode->deleteZeroRows); + } + + return code; +} + +static int32_t jsonToDeleteStmt(const SJson* pJson, void* pObj) { + SDeleteStmt* pNode = (SDeleteStmt*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkDeleteStmtFromTable, &pNode->pFromTable); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkDeleteStmtWhere, &pNode->pWhere); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkDeleteStmtCountFunc, &pNode->pCountFunc); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkDeleteStmtTagIndexCond, &pNode->pTagIndexCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkDeleteStmtTimeRangeStartKey, &pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkDeleteStmtTimeRangeEndKey, &pNode->timeRange.ekey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUTinyIntValue(pJson, jkDeleteStmtPrecision, &pNode->precision); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkDeleteStmtDeleteZeroRows, &pNode->deleteZeroRows); + } + + return code; +} + static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { switch (nodeType(pObj)) { case QUERY_NODE_COLUMN: @@ -3594,6 +3753,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { break; case QUERY_NODE_CREATE_TOPIC_STMT: return createTopicStmtToJson(pObj, pJson); + case QUERY_NODE_DELETE_STMT: + return deleteStmtToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_SCAN: return logicScanNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_JOIN: @@ -3602,8 +3763,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return logicAggNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_PROJECT: return logicProjectNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: - break; + case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: + return logicVnodeModifyNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_EXCHANGE: return logicExchangeNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_MERGE: @@ -3640,7 +3801,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return physiMergeNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SORT: return physiSortNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: @@ -3659,6 +3820,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return physiDispatchNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_INSERT: break; + case QUERY_NODE_PHYSICAL_PLAN_DELETE: + return physiDeleteNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_SUBPLAN: return subplanToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN: @@ -3722,10 +3885,14 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToAlterDnodeStmt(pJson, pObj); case QUERY_NODE_CREATE_TOPIC_STMT: return jsonToCreateTopicStmt(pJson, pObj); + case QUERY_NODE_DELETE_STMT: + return jsonToDeleteStmt(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_SCAN: return jsonToLogicScanNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_PROJECT: return jsonToLogicProjectNode(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: + return jsonToLogicVnodeModifyNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_EXCHANGE: return jsonToLogicExchangeNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_MERGE: @@ -3762,7 +3929,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToPhysiMergeNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_SORT: return jsonToPhysiSortNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: @@ -3779,6 +3946,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToPhysiPartitionNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return jsonToPhysiDispatchNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_DELETE: + return jsonToPhysiDeleteNode(pJson, pObj); case QUERY_NODE_PHYSICAL_SUBPLAN: return jsonToSubplan(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN: diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index e1e809ebd7..d9b61bb002 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -512,7 +512,7 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk } break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext); break; @@ -520,7 +520,7 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION_WINDOW: res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext); break; - case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: + case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW: { SStateWinodwPhysiNode* pState = (SStateWinodwPhysiNode*)pNode; res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 1c17cb3ab6..77732a70cd 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -218,8 +218,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SAggLogicNode)); case QUERY_NODE_LOGIC_PLAN_PROJECT: return makeNode(type, sizeof(SProjectLogicNode)); - case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: - return makeNode(type, sizeof(SVnodeModifLogicNode)); + case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: + return makeNode(type, sizeof(SVnodeModifyLogicNode)); case QUERY_NODE_LOGIC_PLAN_EXCHANGE: return makeNode(type, sizeof(SExchangeLogicNode)); case QUERY_NODE_LOGIC_PLAN_MERGE: @@ -258,8 +258,10 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SMergePhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SORT: return makeNode(type, sizeof(SSortPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: return makeNode(type, sizeof(SIntervalPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL: + return makeNode(type, sizeof(SSortMergeIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: return makeNode(type, sizeof(SStreamIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: @@ -282,6 +284,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SDataDispatcherNode)); case QUERY_NODE_PHYSICAL_PLAN_INSERT: return makeNode(type, sizeof(SDataInserterNode)); + case QUERY_NODE_PHYSICAL_PLAN_DELETE: + return makeNode(type, sizeof(SDataDeleterNode)); case QUERY_NODE_PHYSICAL_SUBPLAN: return makeNode(type, sizeof(SSubplan)); case QUERY_NODE_PHYSICAL_PLAN: @@ -561,7 +565,6 @@ void nodesDestroyNode(SNodeptr pNode) { SScanLogicNode* pLogicNode = (SScanLogicNode*)pNode; destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyList(pLogicNode->pScanCols); - taosMemoryFreeClear(pLogicNode->pMeta); taosMemoryFreeClear(pLogicNode->pVgroupList); break; } @@ -584,8 +587,8 @@ void nodesDestroyNode(SNodeptr pNode) { nodesDestroyList(pLogicNode->pProjections); break; } - case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: { - SVnodeModifLogicNode* pLogicNode = (SVnodeModifLogicNode*)pNode; + case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: { + SVnodeModifyLogicNode* pLogicNode = (SVnodeModifyLogicNode*)pNode; destroyLogicNode((SLogicNode*)pLogicNode); destroyVgDataBlockArray(pLogicNode->pDataBlocks); // pVgDataBlocks is weak reference @@ -673,7 +676,7 @@ void nodesDestroyNode(SNodeptr pNode) { nodesDestroyNode(pPhyNode->pSortKeys); break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); break; @@ -1066,7 +1069,7 @@ char* nodesGetStrValueFromNode(SValueNode* pNode) { bool nodesIsExprNode(const SNode* pNode) { ENodeType type = nodeType(pNode); return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || - QUERY_NODE_FUNCTION == type); + QUERY_NODE_FUNCTION == type || QUERY_NODE_LOGIC_CONDITION == type); } bool nodesIsUnaryOp(const SOperatorNode* pOp) { diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 90c2d7f87a..24db0671ac 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1489,11 +1489,27 @@ SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDb return (SNode*)pStmt; } +SNode* createCountFuncForDelete(SAstCreateContext* pCxt) { + SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION); + CHECK_OUT_OF_MEM(pFunc); + strcpy(pFunc->functionName, "count"); + if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pFunc->pParameterList, createPrimaryKeyCol(pCxt))) { + nodesDestroyNode(pFunc); + CHECK_OUT_OF_MEM(NULL); + } + return (SNode*)pFunc; +} + SNode* createDeleteStmt(SAstCreateContext* pCxt, SNode* pTable, SNode* pWhere) { CHECK_PARSER_STATUS(pCxt); SDeleteStmt* pStmt = nodesMakeNode(QUERY_NODE_DELETE_STMT); CHECK_OUT_OF_MEM(pStmt); pStmt->pFromTable = pTable; pStmt->pWhere = pWhere; + pStmt->pCountFunc = createCountFuncForDelete(pCxt); + if (NULL == pStmt->pCountFunc) { + nodesDestroyNode(pStmt); + CHECK_OUT_OF_MEM(NULL); + } return (SNode*)pStmt; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 27b57bd985..836a3cf8c4 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -938,7 +938,7 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (hasInvalidFuncNesting(pFunc->pParameterList)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING); } - if (pCxt->pCurrSelectStmt->hasIndefiniteRowsFunc) { + if (NULL != pCxt->pCurrSelectStmt && pCxt->pCurrSelectStmt->hasIndefiniteRowsFunc) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } @@ -2124,6 +2124,7 @@ static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) { code = translateDeleteWhere(pCxt, pDelete); } if (TSDB_CODE_SUCCESS == code) { + pCxt->currClause = SQL_CLAUSE_SELECT; code = translateExpr(pCxt, &pDelete->pCountFunc); } return code; diff --git a/source/libs/parser/test/parInitialDTest.cpp b/source/libs/parser/test/parInitialDTest.cpp index 5209021205..d4f9a540ae 100644 --- a/source/libs/parser/test/parInitialDTest.cpp +++ b/source/libs/parser/test/parInitialDTest.cpp @@ -28,6 +28,10 @@ TEST_F(ParserInitialDTest, delete) { run("DELETE FROM t1"); run("DELETE FROM t1 WHERE ts > now - 2d and ts < now - 1d"); + + run("DELETE FROM st1"); + + run("DELETE FROM st1 WHERE ts > now - 2d and ts < now - 1d AND tag1 = 10"); } TEST_F(ParserInitialDTest, deleteSemanticCheck) { diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 464004ae3b..c3aad632c7 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -98,7 +98,15 @@ static int32_t rewriteExprForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESq return cxt.errCode; } +static int32_t rewriteExpr(SNodeList* pExprs, SNode** pTarget) { + nodesWalkExprs(pExprs, doNameExpr, NULL); + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs}; + nodesRewriteExpr(pTarget, doRewriteExpr, &cxt); + return cxt.errCode; +} + static int32_t rewriteExprs(SNodeList* pExprs, SNodeList* pTarget) { + nodesWalkExprs(pExprs, doNameExpr, NULL); SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs}; nodesRewriteExprs(pTarget, doRewriteExpr, &cxt); return cxt.errCode; @@ -141,7 +149,7 @@ static int32_t createSelectRootLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p } static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols, SNodeList* pScanCols, - STableMeta* pMeta) { + int8_t tableType) { if (pCxt->pPlanCxt->topicQuery || pCxt->pPlanCxt->streamQuery) { return SCAN_TYPE_STREAM; } @@ -151,7 +159,7 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols return NULL == pScanPseudoCols ? SCAN_TYPE_TABLE : SCAN_TYPE_TAG; } - if (TSDB_SYSTEM_TABLE == pMeta->tableType) { + if (TSDB_SYSTEM_TABLE == tableType) { return SCAN_TYPE_SYSTEM_TABLE; } @@ -204,16 +212,18 @@ static int32_t addPrimaryKeyCol(uint64_t tableId, SNodeList** pCols) { return TSDB_CODE_SUCCESS; } -static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable, - SLogicNode** pLogicNode) { +static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealTable, bool hasRepeatScanFuncs, + SLogicNode** pLogicNode) { SScanLogicNode* pScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN); if (NULL == pScan) { return TSDB_CODE_OUT_OF_MEMORY; } - TSWAP(pScan->pMeta, pRealTable->pMeta); + // TSWAP(pScan->pMeta, pRealTable->pMeta); TSWAP(pScan->pVgroupList, pRealTable->pVgroupList); - pScan->scanSeq[0] = pSelect->hasRepeatScanFuncs ? 2 : 1; + pScan->tableId = pRealTable->pMeta->uid; + pScan->tableType = pRealTable->pMeta->tableType; + pScan->scanSeq[0] = hasRepeatScanFuncs ? 2 : 1; pScan->scanSeq[1] = 0; pScan->scanRange = TSWINDOW_INITIALIZER; pScan->tableName.type = TSDB_TABLE_NAME_T; @@ -224,9 +234,21 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pScan->ratio = pRealTable->ratio; pScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD; + *pLogicNode = (SLogicNode*)pScan; + + return TSDB_CODE_SUCCESS; +} + +static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable, + SLogicNode** pLogicNode) { + SScanLogicNode* pScan = NULL; + int32_t code = makeScanLogicNode(pCxt, pRealTable, pSelect->hasRepeatScanFuncs, (SLogicNode**)&pScan); + // set columns to scan - int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, COLLECT_COL_TYPE_COL, - &pScan->pScanCols); + if (TSDB_CODE_SUCCESS == code) { + code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, COLLECT_COL_TYPE_COL, + &pScan->pScanCols); + } if (TSDB_CODE_SUCCESS == code) { code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, COLLECT_COL_TYPE_TAG, @@ -242,10 +264,10 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect code = rewriteExprForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM); } - pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->pMeta); + pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType); if (TSDB_CODE_SUCCESS == code) { - code = addPrimaryKeyCol(pScan->pMeta->uid, &pScan->pScanCols); + code = addPrimaryKeyCol(pScan->tableId, &pScan->pScanCols); } // set output @@ -518,7 +540,7 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva pWindow->sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : pWindow->interval); pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); - pWindow->stmInterAlgo = STREAM_INTERVAL_ALGO_SINGLE; + pWindow->intervalAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH; pWindow->pTspk = nodesCloneNode(pInterval->pCol); if (NULL == pWindow->pTspk) { @@ -946,10 +968,11 @@ static int32_t getMsgType(ENodeType sqlType) { } static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt, SLogicNode** pLogicNode) { - SVnodeModifLogicNode* pModif = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIF); + SVnodeModifyLogicNode* pModif = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY); if (NULL == pModif) { return TSDB_CODE_OUT_OF_MEMORY; } + pModif->modifyType = MODIFY_TABLE_TYPE_INSERT; TSWAP(pModif->pDataBlocks, pStmt->pDataBlocks); pModif->msgType = getMsgType(pStmt->sqlNodeType); *pLogicNode = (SLogicNode*)pModif; @@ -962,26 +985,96 @@ static int32_t createDeleteRootLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* p } static int32_t createDeleteScanLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pDelete, SLogicNode** pLogicNode) { - return TSDB_CODE_FAILED; + SScanLogicNode* pScan = NULL; + int32_t code = makeScanLogicNode(pCxt, (SRealTableNode*)pDelete->pFromTable, false, (SLogicNode**)&pScan); + + // set columns to scan + if (TSDB_CODE_SUCCESS == code) { + pScan->scanType = SCAN_TYPE_TABLE; + pScan->pScanCols = nodesCloneList(((SFunctionNode*)pDelete->pCountFunc)->pParameterList); + if (NULL == pScan->pScanCols) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code && NULL != pDelete->pTagIndexCond) { + pScan->pTagCond = nodesCloneNode(pDelete->pTagIndexCond); + if (NULL == pScan->pTagCond) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + // set output + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExps(pScan->pScanCols, &pScan->node.pTargets); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pScan; + } else { + nodesDestroyNode(pScan); + } + + return code; } static int32_t createDeleteAggLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pDelete, SLogicNode** pLogicNode) { - return TSDB_CODE_FAILED; + SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG); + if (NULL == pAgg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = nodesListMakeStrictAppend(&pAgg->pAggFuncs, nodesCloneNode(pDelete->pCountFunc)); + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExpr(pAgg->pAggFuncs, &pDelete->pCountFunc); + } + // set the output + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExps(pAgg->pAggFuncs, &pAgg->node.pTargets); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pAgg; + } else { + nodesDestroyNode(pAgg); + } + + return code; } -static int32_t createDeleteModifyTableLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pDelete, - SLogicNode** pLogicNode) { - return TSDB_CODE_FAILED; +static int32_t createVnodeModifLogicNodeByDelete(SLogicPlanContext* pCxt, SDeleteStmt* pDelete, + SLogicNode** pLogicNode) { + SVnodeModifyLogicNode* pModify = (SVnodeModifyLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY); + if (NULL == pModify) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SRealTableNode* pRealTable = (SRealTableNode*)pDelete->pFromTable; + + pModify->modifyType = MODIFY_TABLE_TYPE_DELETE; + pModify->tableId = pRealTable->pMeta->uid; + pModify->tableType = pRealTable->pMeta->tableType; + snprintf(pModify->tableFName, sizeof(pModify->tableFName), "%d.%s.%s", pCxt->pPlanCxt->acctId, + pRealTable->table.dbName, pRealTable->table.tableName); + pModify->deleteTimeRange = pDelete->timeRange; + pModify->pModifyRows = nodesCloneNode(pDelete->pCountFunc); + if (NULL == pModify->pModifyRows) { + nodesDestroyNode(pModify); + return TSDB_CODE_OUT_OF_MEMORY; + } + + *pLogicNode = (SLogicNode*)pModify; + return TSDB_CODE_SUCCESS; } static int32_t createDeleteLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pDelete, SLogicNode** pLogicNode) { SLogicNode* pRoot = NULL; - int32_t code = createDeleteRootLogicNode(pCxt, pDelete, createDeleteScanLogicNode, &pRoot); + int32_t code = createDeleteScanLogicNode(pCxt, pDelete, &pRoot); if (TSDB_CODE_SUCCESS == code) { code = createDeleteRootLogicNode(pCxt, pDelete, createDeleteAggLogicNode, &pRoot); } if (TSDB_CODE_SUCCESS == code) { - code = createDeleteRootLogicNode(pCxt, pDelete, createDeleteModifyTableLogicNode, &pRoot); + code = createDeleteRootLogicNode(pCxt, pDelete, createVnodeModifLogicNodeByDelete, &pRoot); } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index cae25d5c05..8346aca76f 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -99,7 +99,7 @@ static bool osdMayBeOptimized(SLogicNode* pNode) { return false; } // todo: release after function splitting - if (TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->pMeta->tableType && + if (TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->tableType && SCAN_TYPE_STREAM != ((SScanLogicNode*)pNode)->scanType) { return false; } @@ -328,7 +328,7 @@ static int32_t cpdApplyTagIndex(SScanLogicNode* pScan, SNode** pTagCond, SNode** static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) { if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD) || - TSDB_SYSTEM_TABLE == pScan->pMeta->tableType) { + TSDB_SYSTEM_TABLE == pScan->tableType) { return TSDB_CODE_SUCCESS; } @@ -662,7 +662,7 @@ static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeL switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: - if (TSDB_SUPER_TABLE != ((SScanLogicNode*)pNode)->pMeta->tableType) { + if (TSDB_SUPER_TABLE != ((SScanLogicNode*)pNode)->tableType) { return nodesListMakeAppend(pScanNodes, pNode); } break; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 2734c288f6..3c000e3ff7 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -342,18 +342,7 @@ static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i return TSDB_CODE_SUCCESS; } -static uint8_t getPrecision(SNodeList* pChildren) { - if (1 == LIST_LENGTH(pChildren)) { - return (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc)->precision; - } else if (2 == LIST_LENGTH(pChildren)) { - uint8_t lp = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc)->precision; - uint8_t rp = (((SPhysiNode*)nodesListGetNode(pChildren, 1))->pOutputDataBlockDesc)->precision; - return (lp > rp ? rp : lp); - } - return 0; -} - -static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, uint8_t precision, SLogicNode* pLogicNode, ENodeType type) { +static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, ENodeType type) { SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type); if (NULL == pPhysiNode) { return NULL; @@ -364,7 +353,7 @@ static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, uint8_t precision, SLo nodesDestroyNode(pPhysiNode); return NULL; } - pPhysiNode->pOutputDataBlockDesc->precision = precision; + pPhysiNode->pOutputDataBlockDesc->precision = pLogicNode->precision; return pPhysiNode; } @@ -435,8 +424,8 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS } if (TSDB_CODE_SUCCESS == code) { - pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; - pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; + pScanPhysiNode->uid = pScanLogicNode->tableId; + pScanPhysiNode->tableType = pScanLogicNode->tableType; memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName)); if (NULL != pScanLogicNode->pTagCond) { pSubplan->pTagCond = nodesCloneNode(pScanLogicNode->pTagCond); @@ -462,8 +451,8 @@ static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAdd static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { - STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode( - pCxt, pScanLogicNode->pMeta->tableInfo.precision, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); + STagScanPhysiNode* pTagScan = + (STagScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); if (NULL == pTagScan) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -476,8 +465,7 @@ static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubpla static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { STableScanPhysiNode* pTableScan = - (STableScanPhysiNode*)makePhysiNode(pCxt, pScanLogicNode->pMeta->tableInfo.precision, (SLogicNode*)pScanLogicNode, - QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + (STableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); if (NULL == pTableScan) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -515,9 +503,8 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { - SSystemTableScanPhysiNode* pScan = - (SSystemTableScanPhysiNode*)makePhysiNode(pCxt, pScanLogicNode->pMeta->tableInfo.precision, - (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN); + SSystemTableScanPhysiNode* pScan = (SSystemTableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, + QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN); if (NULL == pScan) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -567,8 +554,8 @@ static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode, SPhysiNode** pPhyNode) { - SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, getPrecision(pChildren), (SLogicNode*)pJoinLogicNode, - QUERY_NODE_PHYSICAL_PLAN_JOIN); + SJoinPhysiNode* pJoin = + (SJoinPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pJoinLogicNode, QUERY_NODE_PHYSICAL_PLAN_JOIN); if (NULL == pJoin) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -750,8 +737,7 @@ static int32_t rewritePrecalcExpr(SPhysiPlanContext* pCxt, SNode* pNode, SNodeLi static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode, SPhysiNode** pPhyNode) { - SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, getPrecision(pChildren), (SLogicNode*)pAggLogicNode, - QUERY_NODE_PHYSICAL_PLAN_AGG); + SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pAggLogicNode, QUERY_NODE_PHYSICAL_PLAN_AGG); if (NULL == pAgg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -806,8 +792,8 @@ static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SProjectLogicNode* pProjectLogicNode, SPhysiNode** pPhyNode) { - SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode( - pCxt, getPrecision(pChildren), (SLogicNode*)pProjectLogicNode, QUERY_NODE_PHYSICAL_PLAN_PROJECT); + SProjectPhysiNode* pProject = + (SProjectPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pProjectLogicNode, QUERY_NODE_PHYSICAL_PLAN_PROJECT); if (NULL == pProject) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -838,8 +824,8 @@ static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChild static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, SPhysiNode** pPhyNode) { - SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode( - pCxt, pExchangeLogicNode->node.precision, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); + SExchangePhysiNode* pExchange = + (SExchangePhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); if (NULL == pExchange) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -852,8 +838,8 @@ static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogic static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, SPhysiNode** pPhyNode) { - SScanPhysiNode* pScan = (SScanPhysiNode*)makePhysiNode( - pCxt, pExchangeLogicNode->node.precision, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + SScanPhysiNode* pScan = + (SScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); if (NULL == pScan) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -933,22 +919,28 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* return code; } -static ENodeType getIntervalOperatorType(bool streamQuery, EStreamIntervalAlgorithm stmAlgo) { - if (streamQuery) { - return STREAM_INTERVAL_ALGO_FINAL == stmAlgo - ? QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL - : (STREAM_INTERVAL_ALGO_SEMI == stmAlgo ? QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL - : QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL); - } else { - return QUERY_NODE_PHYSICAL_PLAN_INTERVAL; +static ENodeType getIntervalOperatorType(EIntervalAlgorithm intervalAlgo) { + switch (intervalAlgo) { + case INTERVAL_ALGO_HASH: + return QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL; + case INTERVAL_ALGO_SORT_MERGE: + return QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL; + case INTERVAL_ALGO_STREAM_FINAL: + return QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL; + case INTERVAL_ALGO_STREAM_SEMI: + return QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL; + case INTERVAL_ALGO_STREAM_SINGLE: + return QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL; + default: + break; } + return QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL; } static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { SIntervalPhysiNode* pInterval = (SIntervalPhysiNode*)makePhysiNode( - pCxt, getPrecision(pChildren), (SLogicNode*)pWindowLogicNode, - getIntervalOperatorType(pCxt->pPlanCxt->streamQuery, pWindowLogicNode->stmInterAlgo)); + pCxt, (SLogicNode*)pWindowLogicNode, getIntervalOperatorType(pWindowLogicNode->intervalAlgo)); if (NULL == pInterval) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -965,7 +957,7 @@ static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChil static int32_t createSessionWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { SSessionWinodwPhysiNode* pSession = (SSessionWinodwPhysiNode*)makePhysiNode( - pCxt, getPrecision(pChildren), (SLogicNode*)pWindowLogicNode, + pCxt, (SLogicNode*)pWindowLogicNode, (pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION_WINDOW : QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW)); if (NULL == pSession) { @@ -979,10 +971,10 @@ static int32_t createSessionWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* static int32_t createStateWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { - SStateWinodwPhysiNode* pState = (SStateWinodwPhysiNode*)makePhysiNode( - pCxt, getPrecision(pChildren), (SLogicNode*)pWindowLogicNode, - (pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW: - QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW)); + SStateWinodwPhysiNode* pState = + (SStateWinodwPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pWindowLogicNode, + (pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW + : QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW)); if (NULL == pState) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1032,8 +1024,8 @@ static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildr static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SSortLogicNode* pSortLogicNode, SPhysiNode** pPhyNode) { - SSortPhysiNode* pSort = (SSortPhysiNode*)makePhysiNode(pCxt, getPrecision(pChildren), (SLogicNode*)pSortLogicNode, - QUERY_NODE_PHYSICAL_PLAN_SORT); + SSortPhysiNode* pSort = + (SSortPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pSortLogicNode, QUERY_NODE_PHYSICAL_PLAN_SORT); if (NULL == pSort) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1073,8 +1065,8 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SPartitionLogicNode* pPartLogicNode, SPhysiNode** pPhyNode) { - SPartitionPhysiNode* pPart = (SPartitionPhysiNode*)makePhysiNode( - pCxt, getPrecision(pChildren), (SLogicNode*)pPartLogicNode, QUERY_NODE_PHYSICAL_PLAN_PARTITION); + SPartitionPhysiNode* pPart = + (SPartitionPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pPartLogicNode, QUERY_NODE_PHYSICAL_PLAN_PARTITION); if (NULL == pPart) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1114,8 +1106,7 @@ static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChi static int32_t createFillPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SFillLogicNode* pFillNode, SPhysiNode** pPhyNode) { - SFillPhysiNode* pFill = (SFillPhysiNode*)makePhysiNode(pCxt, getPrecision(pChildren), (SLogicNode*)pFillNode, - QUERY_NODE_PHYSICAL_PLAN_FILL); + SFillPhysiNode* pFill = (SFillPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pFillNode, QUERY_NODE_PHYSICAL_PLAN_FILL); if (NULL == pFill) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1168,8 +1159,8 @@ static int32_t createExchangePhysiNodeByMerge(SMergePhysiNode* pMerge) { } static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pMergeLogicNode, SPhysiNode** pPhyNode) { - SMergePhysiNode* pMerge = (SMergePhysiNode*)makePhysiNode( - pCxt, pMergeLogicNode->node.precision, (SLogicNode*)pMergeLogicNode, QUERY_NODE_PHYSICAL_PLAN_MERGE); + SMergePhysiNode* pMerge = + (SMergePhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pMergeLogicNode, QUERY_NODE_PHYSICAL_PLAN_MERGE); if (NULL == pMerge) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1308,6 +1299,62 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl return pSubplan; } +static int32_t buildInsertSubplan(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, SSubplan* pSubplan) { + pSubplan->msgType = pModify->msgType; + pSubplan->execNode.epSet = pModify->pVgDataBlocks->vg.epSet; + SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0}; + taosArrayPush(pCxt->pExecNodeList, &node); + return createDataInserter(pCxt, pModify->pVgDataBlocks, &pSubplan->pDataSink); +} + +static int32_t createDataDeleter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, const SPhysiNode* pRoot, + SDataSinkNode** pSink) { + SDataDeleterNode* pDeleter = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_DELETE); + if (NULL == pDeleter) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pDeleter->tableId = pModify->tableId; + pDeleter->tableType = pModify->tableType; + strcpy(pDeleter->tableFName, pModify->tableFName); + pDeleter->deleteTimeRange = pModify->deleteTimeRange; + + pDeleter->sink.pInputDataBlockDesc = nodesCloneNode(pRoot->pOutputDataBlockDesc); + if (NULL == pDeleter->sink.pInputDataBlockDesc) { + nodesDestroyNode(pDeleter); + return TSDB_CODE_OUT_OF_MEMORY; + } + + *pSink = (SDataSinkNode*)pDeleter; + return TSDB_CODE_SUCCESS; +} + +static int32_t buildDeleteSubplan(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, SSubplan* pSubplan) { + int32_t code = + createPhysiNode(pCxt, (SLogicNode*)nodesListGetNode(pModify->node.pChildren, 0), pSubplan, &pSubplan->pNode); + if (TSDB_CODE_SUCCESS == code) { + code = createDataDeleter(pCxt, pModify, pSubplan->pNode, &pSubplan->pDataSink); + } + return code; +} + +static int32_t buildVnodeModifySubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan* pSubplan) { + int32_t code = TSDB_CODE_SUCCESS; + SVnodeModifyLogicNode* pModify = (SVnodeModifyLogicNode*)pLogicSubplan->pNode; + switch (pModify->modifyType) { + case MODIFY_TABLE_TYPE_INSERT: + code = buildInsertSubplan(pCxt, pModify, pSubplan); + break; + case MODIFY_TABLE_TYPE_DELETE: + code = buildDeleteSubplan(pCxt, pModify, pSubplan); + break; + default: + code = TSDB_CODE_FAILED; + break; + } + return code; +} + static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan** pPhysiSubplan) { SSubplan* pSubplan = makeSubplan(pCxt, pLogicSubplan); if (NULL == pSubplan) { @@ -1317,12 +1364,7 @@ static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogic int32_t code = TSDB_CODE_SUCCESS; if (SUBPLAN_TYPE_MODIFY == pLogicSubplan->subplanType) { - SVnodeModifLogicNode* pModif = (SVnodeModifLogicNode*)pLogicSubplan->pNode; - pSubplan->msgType = pModif->msgType; - pSubplan->execNode.epSet = pModif->pVgDataBlocks->vg.epSet; - SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0}; - taosArrayPush(pCxt->pExecNodeList, &node); - code = createDataInserter(pCxt, pModif->pVgDataBlocks, &pSubplan->pDataSink); + code = buildVnodeModifySubplan(pCxt, pLogicSubplan, pSubplan); } else { pSubplan->msgType = TDMT_VND_QUERY; code = createPhysiNode(pCxt, pLogicSubplan->pNode, pSubplan, &pSubplan->pNode); diff --git a/source/libs/planner/src/planScaleOut.c b/source/libs/planner/src/planScaleOut.c index 73c8666673..5dd016afb8 100644 --- a/source/libs/planner/src/planScaleOut.c +++ b/source/libs/planner/src/planScaleOut.c @@ -38,26 +38,6 @@ static SLogicSubplan* singleCloneSubLogicPlan(SScaleOutContext* pCxt, SLogicSubp return pDst; } -static int32_t scaleOutForModify(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { - SVnodeModifLogicNode* pNode = (SVnodeModifLogicNode*)pSubplan->pNode; - size_t numOfVgroups = taosArrayGetSize(pNode->pDataBlocks); - for (int32_t i = 0; i < numOfVgroups; ++i) { - SLogicSubplan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); - if (NULL == pNewSubplan) { - return TSDB_CODE_OUT_OF_MEMORY; - } - ((SVnodeModifLogicNode*)pNewSubplan->pNode)->pVgDataBlocks = (SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i); - if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pGroup, pNewSubplan)) { - return TSDB_CODE_OUT_OF_MEMORY; - } - } - return TSDB_CODE_SUCCESS; -} - -static int32_t scaleOutForMerge(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { - return nodesListStrictAppend(pGroup, singleCloneSubLogicPlan(pCxt, pSubplan, level)); -} - static int32_t doSetScanVgroup(SLogicNode* pNode, const SVgroupInfo* pVgroup, bool* pFound) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { SScanLogicNode* pScan = (SScanLogicNode*)pNode; @@ -84,23 +64,52 @@ static int32_t setScanVgroup(SLogicNode* pNode, const SVgroupInfo* pVgroup) { return doSetScanVgroup(pNode, pVgroup, &found); } -static int32_t scaleOutForScan(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { - if (pSubplan->pVgroupList && !pCxt->pPlanCxt->streamQuery) { - int32_t code = TSDB_CODE_SUCCESS; - for (int32_t i = 0; i < pSubplan->pVgroupList->numOfVgroups; ++i) { +static int32_t scaleOutByVgroups(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { + int32_t code = TSDB_CODE_SUCCESS; + for (int32_t i = 0; i < pSubplan->pVgroupList->numOfVgroups; ++i) { + SLogicSubplan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); + if (NULL == pNewSubplan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + code = setScanVgroup(pNewSubplan->pNode, pSubplan->pVgroupList->vgroups + i); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pGroup, pNewSubplan); + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + return code; +} + +static int32_t scaleOutForModify(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { + SVnodeModifyLogicNode* pNode = (SVnodeModifyLogicNode*)pSubplan->pNode; + if (MODIFY_TABLE_TYPE_DELETE == pNode->modifyType) { + return scaleOutByVgroups(pCxt, pSubplan, level, pGroup); + } else { + size_t numOfVgroups = taosArrayGetSize(pNode->pDataBlocks); + for (int32_t i = 0; i < numOfVgroups; ++i) { SLogicSubplan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); if (NULL == pNewSubplan) { return TSDB_CODE_OUT_OF_MEMORY; } - code = setScanVgroup(pNewSubplan->pNode, pSubplan->pVgroupList->vgroups + i); - if (TSDB_CODE_SUCCESS == code) { - code = nodesListStrictAppend(pGroup, pNewSubplan); - } - if (TSDB_CODE_SUCCESS != code) { - break; + ((SVnodeModifyLogicNode*)pNewSubplan->pNode)->pVgDataBlocks = + (SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i); + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pGroup, pNewSubplan)) { + return TSDB_CODE_OUT_OF_MEMORY; } } - return code; + return TSDB_CODE_SUCCESS; + } +} + +static int32_t scaleOutForMerge(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { + return nodesListStrictAppend(pGroup, singleCloneSubLogicPlan(pCxt, pSubplan, level)); +} + +static int32_t scaleOutForScan(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { + if (pSubplan->pVgroupList && !pCxt->pPlanCxt->streamQuery) { + return scaleOutByVgroups(pCxt, pSubplan, level, pGroup); } else { return scaleOutForMerge(pCxt, pSubplan, level, pGroup); } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index a3d5086908..9e6c25ce78 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -138,7 +138,7 @@ static bool stbSplHasGatherExecFunc(const SNodeList* pFuncs) { static bool stbSplIsMultiTbScan(bool streamQuery, SScanLogicNode* pScan) { return (NULL != pScan->pVgroupList && pScan->pVgroupList->numOfVgroups > 1) || - (streamQuery && TSDB_SUPER_TABLE == pScan->pMeta->tableType); + (streamQuery && TSDB_SUPER_TABLE == pScan->tableType); } static bool stbSplHasMultiTbScan(bool streamQuery, SLogicNode* pNode) { @@ -315,27 +315,6 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicNode* pParent, S return nodesListMakeAppend(&pParent->pChildren, pMerge); } -static int32_t stbSplSplitWindowNodeForBatch(SSplitContext* pCxt, SStableSplitInfo* pInfo) { - SLogicNode* pPartWindow = NULL; - int32_t code = stbSplCreatePartWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow); - if (TSDB_CODE_SUCCESS == code) { - SNodeList* pMergeKeys = NULL; - code = nodesListMakeStrictAppend(&pMergeKeys, nodesCloneNode(((SWindowLogicNode*)pInfo->pSplitNode)->pTspk)); - if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, pInfo->pSplitNode, pMergeKeys, pPartWindow); - } - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyList(pMergeKeys); - } - } - if (TSDB_CODE_SUCCESS == code) { - code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, - splCreateScanSubplan(pCxt, pPartWindow, SPLIT_FLAG_STABLE_SPLIT)); - } - pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; - return code; -} - static int32_t stbSplCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pParent, SLogicNode* pPartChild) { SExchangeLogicNode* pExchange = NULL; int32_t code = splCreateExchangeNode(pCxt, pPartChild, &pExchange); @@ -345,12 +324,28 @@ static int32_t stbSplCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pParent return code; } +static int32_t stbSplSplitWindowNodeForBatch(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + SLogicNode* pPartWindow = NULL; + int32_t code = stbSplCreatePartWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow); + if (TSDB_CODE_SUCCESS == code) { + ((SWindowLogicNode*)pPartWindow)->intervalAlgo = INTERVAL_ALGO_HASH; + ((SWindowLogicNode*)pInfo->pSplitNode)->intervalAlgo = INTERVAL_ALGO_SORT_MERGE; + code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, + splCreateScanSubplan(pCxt, pPartWindow, SPLIT_FLAG_STABLE_SPLIT)); + } + pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + return code; +} + static int32_t stbSplSplitWindowNodeForStream(SSplitContext* pCxt, SStableSplitInfo* pInfo) { SLogicNode* pPartWindow = NULL; int32_t code = stbSplCreatePartWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow); if (TSDB_CODE_SUCCESS == code) { - ((SWindowLogicNode*)pPartWindow)->stmInterAlgo = STREAM_INTERVAL_ALGO_SEMI; - ((SWindowLogicNode*)pInfo->pSplitNode)->stmInterAlgo = STREAM_INTERVAL_ALGO_FINAL; + ((SWindowLogicNode*)pPartWindow)->intervalAlgo = INTERVAL_ALGO_STREAM_SEMI; + ((SWindowLogicNode*)pInfo->pSplitNode)->intervalAlgo = INTERVAL_ALGO_STREAM_FINAL; code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow); } if (TSDB_CODE_SUCCESS == code) { @@ -834,6 +829,16 @@ static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) { static void setLogicNodeParent(SLogicNode* pNode) { doSetLogicNodeParent(pNode, NULL); } +static void setVgroupsInfo(SLogicNode* pNode, SLogicSubplan* pSubplan) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + TSWAP(((SScanLogicNode*)pNode)->pVgroupList, pSubplan->pVgroupList); + return; + } + + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { setVgroupsInfo((SLogicNode*)pChild, pSubplan); } +} + int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan** pLogicSubplan) { SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); if (NULL == pSubplan) { @@ -845,17 +850,21 @@ int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan nodesDestroyNode(pSubplan); return TSDB_CODE_OUT_OF_MEMORY; } - if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIF == nodeType(pLogicNode)) { - pSubplan->subplanType = SUBPLAN_TYPE_MODIFY; - TSWAP(((SVnodeModifLogicNode*)pLogicNode)->pDataBlocks, ((SVnodeModifLogicNode*)pSubplan->pNode)->pDataBlocks); - } else { - pSubplan->subplanType = SUBPLAN_TYPE_SCAN; - } + pSubplan->id.queryId = pCxt->queryId; pSubplan->id.groupId = 1; setLogicNodeParent(pSubplan->pNode); - int32_t code = applySplitRule(pCxt, pSubplan); + int32_t code = TSDB_CODE_SUCCESS; + if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY == nodeType(pLogicNode)) { + pSubplan->subplanType = SUBPLAN_TYPE_MODIFY; + TSWAP(((SVnodeModifyLogicNode*)pLogicNode)->pDataBlocks, ((SVnodeModifyLogicNode*)pSubplan->pNode)->pDataBlocks); + setVgroupsInfo(pSubplan->pNode, pSubplan); + } else { + pSubplan->subplanType = SUBPLAN_TYPE_SCAN; + code = applySplitRule(pCxt, pSubplan); + } + if (TSDB_CODE_SUCCESS == code) { *pLogicSubplan = pSubplan; } else { diff --git a/source/libs/planner/test/planOtherTest.cpp b/source/libs/planner/test/planOtherTest.cpp index f153604e6b..d5d37fda64 100644 --- a/source/libs/planner/test/planOtherTest.cpp +++ b/source/libs/planner/test/planOtherTest.cpp @@ -60,3 +60,15 @@ TEST_F(PlanOtherTest, show) { run("SHOW DATABASES"); } + +TEST_F(PlanOtherTest, delete) { + useDb("root", "test"); + + run("DELETE FROM t1"); + + run("DELETE FROM t1 WHERE ts > now - 2d and ts < now - 1d"); + + run("DELETE FROM st1"); + + run("DELETE FROM st1 WHERE ts > now - 2d and ts < now - 1d AND tag1 = 10"); +} From 36fd0dac68fd72b6361fc3a19f227c5a2ab67250 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 5 Jun 2022 16:34:06 +0800 Subject: [PATCH 41/63] refactor: add alter-confirm while alter db --- include/util/taoserror.h | 6 ++--- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 28 ++++++++++++++------- source/dnode/mnode/impl/src/mndTrans.c | 9 ++++++- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- tests/script/tsim/db/alter_replica_31.sim | 21 ++++++++++++++++ 5 files changed, 52 insertions(+), 14 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 2d016cab9e..cbef0b46a7 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -40,9 +40,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error //common & util -#define TSDB_CODE_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) -#define TSDB_CODE_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0002) -#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0003) +#define TSDB_CODE_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0003) +#define TSDB_CODE_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0004) +#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) #define TSDB_CODE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0010) #define TSDB_CODE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0011) #define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x0012) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index b1c04fd8cf..427b3d5c94 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -118,25 +118,36 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO pMsg = *(SRpcMsg **)taosArrayGet(pArray, m); code = vnodePreprocessReq(pVnode->pImpl, pMsg); - if (code == TSDB_CODE_ACTION_IN_PROGRESS) continue; - if (code != 0) { - dError("vgId:%d, msg:%p failed to write since %s", pVnode->vgId, pMsg, tstrerror(code)); - vmSendRsp(pMsg, code); + if (code == TSDB_CODE_ACTION_IN_PROGRESS) { + dTrace("vgId:%d, msg:%p in progress and no rsp", pVnode->vgId, pMsg); continue; } - code = syncPropose(sync, pMsg, false); + if (pMsg->msgType != TDMT_VND_ALTER_REPLICA) { + code = syncPropose(sync, pMsg, false); + } + if (code == TAOS_SYNC_PROPOSE_SUCCESS) { + dTrace("vgId:%d, msg:%p is proposed and no rsp", pVnode->vgId, pMsg); continue; } else if (code == TAOS_SYNC_PROPOSE_NOT_LEADER) { - dTrace("vgId:%d, msg:%p is redirect since not leader", pVnode->vgId, pMsg); SEpSet newEpSet = {0}; syncGetEpSet(sync, &newEpSet); - newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps; + SEp *pEp = &newEpSet.eps[newEpSet.inUse]; + if (pEp->port == tsServerPort && strcmp(pEp->fqdn, tsLocalFqdn) == 0) { + newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps; + } + + dTrace("vgId:%d, msg:%p is redirect since not leader, numOfEps:%d inUse:%d", pVnode->vgId, pMsg, + newEpSet.numOfEps, newEpSet.inUse); + for (int32_t i = 0; i < newEpSet.numOfEps; ++i) { + dTrace("vgId:%d, msg:%p ep:%s:%u", pVnode->vgId, pMsg, newEpSet.eps[i].fqdn, newEpSet.eps[i].port); + } + SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info}; tmsgSendRedirectRsp(&rsp, &newEpSet); } else { - dError("vgId:%d, msg:%p failed to write since %s", pVnode->vgId, pMsg, tstrerror(code)); + dError("vgId:%d, msg:%p failed to propose write since %s, code:0x%x", pVnode->vgId, pMsg, tstrerror(code), code); vmSendRsp(pMsg, code); } } @@ -251,7 +262,6 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp switch (qtype) { case QUERY_QUEUE: vnodePreprocessQueryMsg(pVnode->pImpl, pMsg); - dTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); taosWriteQitem(pVnode->pQueryQ, pMsg); break; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index cc9cc6edc2..8c0ba65fb2 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -870,8 +870,15 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) pAction->rawWritten = 0; pAction->msgSent = 0; pAction->msgReceived = 0; + if (pAction->errCode == TSDB_CODE_RPC_REDIRECT) { + pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps; + mDebug("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage), + action, pAction->epSet.inUse); + } else { + mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action); + } pAction->errCode = 0; - mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action); + } } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 37f765d786..8f98e9cec5 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -69,7 +69,7 @@ int32_t vnodeSyncAlter(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = syncReconfig(pVnode->sync, &cfg); if (code == TAOS_SYNC_PROPOSE_SUCCESS) { // todo refactor - SRpcMsg rsp = {.info = pMsg->info, .code = terrno}; + SRpcMsg rsp = {.info = pMsg->info, .code = 0}; tmsgSendRsp(&rsp); return TSDB_CODE_ACTION_IN_PROGRESS; } diff --git a/tests/script/tsim/db/alter_replica_31.sim b/tests/script/tsim/db/alter_replica_31.sim index 896d6708b7..cbd4183b5e 100644 --- a/tests/script/tsim/db/alter_replica_31.sim +++ b/tests/script/tsim/db/alter_replica_31.sim @@ -78,6 +78,15 @@ if $data(4)[2] != 1 then endi # v1_dnode +$hasleader = 0 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 20 then + print ====> dnode not ready! + return -1 + endi sql show db.vgroups print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 if $data(2)[3] != 4 then @@ -89,6 +98,18 @@ endi if $data(2)[7] != 2 then return -1 endi +if $data(2)[4] == leader then + $hasleader = 1 +endi +if $data(2)[6] == leader then + $hasleader = 1 +endi +if $data(2)[8] == leader then + $hasleader = 1 +endi +if $hasleader != 1 then + goto step2 +endi sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd" sql create table db.ctb using db.stb tags(101, "102") From c59b52855b8ebce9ec1bd37b8c4dcd5188a1fe8f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 5 Jun 2022 08:42:18 +0000 Subject: [PATCH 42/63] more refact --- source/dnode/vnode/src/inc/tsdb.h | 4 +- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 87 +++++++++++------------ source/dnode/vnode/src/tsdb/tsdbFile.c | 2 +- 3 files changed, 45 insertions(+), 48 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 02b9e4fcff..5ffa367ff7 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -263,12 +263,12 @@ struct TSDBKEY { }; typedef struct { + uint64_t suid; + uint64_t uid; uint32_t len; uint32_t offset; uint32_t hasLast : 2; uint32_t numOfBlocks : 30; - uint64_t suid; - uint64_t uid; TSDBKEY maxKey; } SBlockIdx; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 6c24396810..321667c55b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -20,6 +20,9 @@ typedef struct { int32_t minutes; int8_t precision; TSKEY nCommitKey; + int32_t fid; + TSKEY minKey; + TSKEY maxKey; SReadH readh; SDFileSet wSet; SArray *aBlkIdx; @@ -180,11 +183,11 @@ static int32_t tsdbCommitTable(SCommitH *pCHandle, SMemData *pMemData, SBlockIdx // commit table impl if (pMemData && pBlockIdx) { - // merge + // TODO } else if (pMemData) { - // new one + // TODO } else { - // save old ones + // TODO } // commit table end @@ -224,9 +227,13 @@ static int32_t tsdbWriteBlockIdx(SDFile *pFile, SArray *pArray, uint8_t **ppBuf) return code; } -static int32_t tsdbCommitFileStart(SCommitH *pCHandle, int32_t fid) { - int32_t code = 0; - // TODO +static int32_t tsdbCommitFileStart(SCommitH *pCHandle) { + int32_t code = 0; + STsdb *pTsdb = pCHandle->pMemTable->pTsdb; + SDFileSet *pSet = NULL; + + taosArrayClear(pCHandle->aBlkIdx); + return code; } @@ -236,61 +243,50 @@ static int32_t tsdbCommitFileEnd(SCommitH *pCHandle) { return code; } -static int32_t tsdbCommitFile(SCommitH *pCHandle, int32_t fid) { - int32_t code = 0; - SMemDataIter iter = {0}; - TSDBROW *pRow = NULL; - int8_t hasData = 0; - TSKEY fidSKey; - TSKEY fidEKey; - int32_t iMemData; - int32_t nMemData; - int32_t iBlockIdx; - int32_t nBlockIdx; +static int32_t tsdbCommitFile(SCommitH *pCHandle) { + int32_t code = 0; + SMemData *pMemData; + SBlockIdx *pBlockIdx; + int32_t iMemData; + int32_t nMemData; + int32_t iBlockIdx; + int32_t nBlockIdx; - pCHandle->nCommitKey = TSKEY_MAX; - - // create or open the file to commit(todo) - code = tsdbCommitFileStart(pCHandle, fid); + // commit file start + code = tsdbCommitFileStart(pCHandle); if (code) { goto _err; } - // loop to commit each table data + // commit file impl iMemData = 0; nMemData = taosArrayGetSize(pCHandle->pMemTable->aMemData); iBlockIdx = 0; - nBlockIdx = 0; - for (;;) { - if (iBlockIdx >= nBlockIdx && iMemData >= nMemData) { - // code = tsdbWriteBlockIdx(); - // if (code) { - // goto _err; - // } - break; - } + nBlockIdx = 0; // todo - SMemData *pMemData = NULL; - SBlockIdx *pBlockIdx = NULL; + for (;;) { + if (iMemData >= nMemData && iBlockIdx >= nBlockIdx) break; + + pMemData = NULL; + pBlockIdx = NULL; if (iMemData < nMemData) { - pMemData = (SMemData *)taosArrayGetP(pCHandle->pMemTable->aMemData, iBlockIdx); + pMemData = (SMemData *)taosArrayGetP(pCHandle->pMemTable->aMemData, iMemData); } if (iBlockIdx < nBlockIdx) { - // pBlockIdx + // pBlockIdx = ; } if (pMemData && pBlockIdx) { - int32_t c = tsdbTableIdCmprFn(&(TABLEID){.suid = pMemData->suid, .uid = pMemData->uid}, - &(TABLEID){.suid = pBlockIdx->suid, .uid = pBlockIdx->uid}); - if (c == 0) { + int32_t c = tsdbTableIdCmprFn(pMemData, pBlockIdx); + if (c < 0) { iMemData++; - iBlockIdx++; - } else if (c < 0) { pBlockIdx = NULL; + } else if (c == 0) { iMemData++; - } else { - pMemData = NULL; iBlockIdx++; + } else { + iBlockIdx++; + pMemData = NULL; } } else { if (pMemData) { @@ -306,7 +302,7 @@ static int32_t tsdbCommitFile(SCommitH *pCHandle, int32_t fid) { } } - // close file + // commit file end code = tsdbCommitFileEnd(pCHandle); if (code) { goto _err; @@ -328,8 +324,9 @@ static int32_t tsdbCommitData(SCommitH *pCHandle) { for (;;) { if (pCHandle->nCommitKey == TSKEY_MAX) break; - fid = TSDB_KEY_FID(pCHandle->nCommitKey, pCHandle->minutes, pCHandle->precision); - code = tsdbCommitFile(pCHandle, fid); + pCHandle->fid = TSDB_KEY_FID(pCHandle->nCommitKey, pCHandle->minutes, pCHandle->precision); + tsdbGetFidKeyRange(pCHandle->minutes, pCHandle->precision, pCHandle->fid, &pCHandle->minKey, &pCHandle->maxKey); + code = tsdbCommitFile(pCHandle); if (code) { goto _err; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 4198a94655..e7c8c7ccd9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -25,7 +25,7 @@ static const char *TSDB_FNAME_SUFFIX[] = { "meta", // TSDB_FILE_META }; -static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char* dname, char *fname); +static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char *dname, char *fname); // static int tsdbRollBackMFile(SMFile *pMFile); static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo); static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo); From 8c75980679f68b7b87a189cdd273da06f0c09e1b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 5 Jun 2022 08:46:10 +0000 Subject: [PATCH 43/63] more refact --- source/dnode/vnode/src/inc/tsdb.h | 4 +--- source/dnode/vnode/src/tsdb/tsdbFS.c | 11 ----------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 5ffa367ff7..94e0a8cd07 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -189,9 +189,7 @@ typedef struct { struct STsdbFS { TdThreadRwlock lock; - SFSStatus *cstatus; // current status - SHashObj *metaCache; // meta cache - SHashObj *metaCacheComp; // meta cache for compact + SFSStatus *cstatus; // current status bool intxn; SFSStatus *nstatus; // new status }; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 584a1ec49c..19a79ac5c4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -216,16 +216,7 @@ STsdbFS *tsdbNewFS(const STsdbKeepCfg *pCfg) { return NULL; } - pfs->metaCache = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (pfs->metaCache == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbFreeFS(pfs); - return NULL; - } - pfs->intxn = false; - pfs->metaCacheComp = NULL; - pfs->nstatus = tsdbNewFSStatus(maxFSet); if (pfs->nstatus == NULL) { tsdbFreeFS(pfs); @@ -238,8 +229,6 @@ STsdbFS *tsdbNewFS(const STsdbKeepCfg *pCfg) { void *tsdbFreeFS(STsdbFS *pfs) { if (pfs) { pfs->nstatus = tsdbFreeFSStatus(pfs->nstatus); - taosHashCleanup(pfs->metaCache); - pfs->metaCache = NULL; pfs->cstatus = tsdbFreeFSStatus(pfs->cstatus); taosThreadRwlockDestroy(&(pfs->lock)); taosMemoryFree(pfs); From c9daa37886a1fff79264f6e0a2fe98ec30edba50 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 5 Jun 2022 16:52:28 +0800 Subject: [PATCH 44/63] feat: add sort/group logic for json --- source/libs/scalar/src/sclvector.c | 38 ++-- .../libs/scalar/test/scalar/scalarTests.cpp | 164 ++++++++++++++---- tests/system-test/2-query/json_tag.py | 2 +- 3 files changed, 144 insertions(+), 60 deletions(-) diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 9b3a8261cc..85778e6a35 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1237,11 +1237,10 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); + _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); + _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); double *output = (double *)pOutputCol->pData; - double zero = 0.0; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { @@ -1250,18 +1249,18 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam continue; } - double lx = getVectorDoubleValueFnLeft(LEFT_COL, i); - double rx = getVectorDoubleValueFnRight(RIGHT_COL, i); - if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) { + int64_t lx = getVectorBigintValueFnLeft(LEFT_COL, i); + int64_t rx = getVectorBigintValueFnRight(RIGHT_COL, i); + if (rx == 0) { colDataAppendNULL(pOutputCol, i); continue; } - *output = lx - ((int64_t)(lx / rx)) * rx; + *output = lx % rx; } } else if (pLeft->numOfRows == 1) { - double lx = getVectorDoubleValueFnLeft(LEFT_COL, 0); - if (IS_HELPER_NULL(pLeftCol, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value + int64_t lx = getVectorBigintValueFnLeft(LEFT_COL, 0); + if (IS_HELPER_NULL(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); } else { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { @@ -1270,18 +1269,18 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam continue; } - double rx = getVectorDoubleValueFnRight(RIGHT_COL, i); - if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) { + int64_t rx = getVectorBigintValueFnRight(RIGHT_COL, i); + if (rx == 0){ colDataAppendNULL(pOutputCol, i); continue; } - *output = lx - ((int64_t)(lx / rx)) * rx; + *output = lx % rx; } } } else if (pRight->numOfRows == 1) { - double rx = getVectorDoubleValueFnRight(RIGHT_COL, 0); - if (IS_HELPER_NULL(pRightCol, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value + int64_t rx = getVectorBigintValueFnRight(RIGHT_COL, 0); + if (IS_HELPER_NULL(pRightCol, 0) || rx == 0) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); } else { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { @@ -1290,13 +1289,8 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam continue; } - double lx = getVectorDoubleValueFnLeft(LEFT_COL, i); - if (isnan(lx) || isinf(lx)) { - colDataAppendNULL(pOutputCol, i); - continue; - } - - *output = lx - ((int64_t)(lx / rx)) * rx; + int64_t lx = getVectorBigintValueFnLeft(LEFT_COL, i); + *output = lx % rx; } } } @@ -1402,7 +1396,7 @@ static void vectorBitAndHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRigh _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); - double *output = (double *)pOutputCol->pData; + int64_t *output = (int64_t *)pOutputCol->pData; if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 8a29462a2b..9519c7d541 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -212,6 +212,24 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in *pNode = (SNode *)rnode; } +void scltMakeOpNode2(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight, bool isReverse) { + SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_OPERATOR); + SOperatorNode *onode = (SOperatorNode *)node; + onode->node.resType.type = resType; + onode->node.resType.bytes = tDataTypes[resType].bytes; + + onode->opType = opType; + if(isReverse){ + onode->pLeft = pRight; + onode->pRight = pLeft; + }else{ + onode->pLeft = pLeft; + onode->pRight = pRight; + } + + *pNode = (SNode *)onode; +} + void scltMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight) { SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_OPERATOR); SOperatorNode *onode = (SOperatorNode *)node; @@ -1039,7 +1057,7 @@ void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){ scltMakeOpNode(opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight); } -void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32_t rightType, void *rightData){ +void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32_t rightType, void *rightData, bool isReverse){ int32_t resType = TSDB_DATA_TYPE_NULL; if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || opType == OP_TYPE_DIV || opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){ @@ -1057,7 +1075,7 @@ void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32 SNode *right = NULL; scltMakeValueNode(&right, rightType, rightData); - scltMakeOpNode(opNode, opType, resType, *opNode, right); + scltMakeOpNode2(opNode, opType, resType, *opNode, right, isReverse); SColumnInfo colInfo = createColumnInfo(1, resType, tDataTypes[resType].bytes); int16_t dataBlockId = 0, slotId = 0; @@ -1065,7 +1083,7 @@ void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32 scltMakeTargetNode(opNode, dataBlockId, slotId, *opNode); } -void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, double exceptValue, EOperatorType opType){ +void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, double exceptValue, EOperatorType opType, bool isReverse){ SArray *blockList = taosArrayInit(2, POINTER_BYTES); SSDataBlock *src = NULL; SNode *opNode = NULL; @@ -1073,7 +1091,7 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do makeJsonArrow(&src, &opNode, json, (char*)key); taosArrayPush(blockList, &src); - makeOperator(&opNode, blockList, opType, rightType, rightData); + makeOperator(&opNode, blockList, opType, rightType, rightData, isReverse); int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); @@ -1107,7 +1125,7 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do TEST(columnTest, json_column_arith_op) { scltInitLogFile(); - char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}"; + char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":-10,\"k7\":-9.8,\"k8\":false,\"k9\":\"8hel\"}"; char rightv[256] = {0}; memcpy(rightv, rightvTmp, strlen(rightvTmp)); @@ -1120,51 +1138,123 @@ TEST(columnTest, json_column_arith_op) { OP_TYPE_MOD, OP_TYPE_MINUS, OP_TYPE_BIT_AND, OP_TYPE_BIT_OR}; int32_t input[len] = {1, 8, 2, 2, 3, 0, -4, 9}; - printf("--------------------json int---------------------\n"); + printf("--------------------json int-4 op {1, 8, 2, 2, 3, 0, -4, 9}--------------------\n"); char *key = "k1"; - double eRes[len] = {5.0, -4, 8.0, 2.0, 1.0, -4, 4&-4, 4|9}; + double eRes00[len] = {5.0, -4, 8.0, 2.0, 1.0, -4, 4&-4, 4|9}; + double eRes01[len] = {5.0, 4, 8.0, 0.5, 3, 0, 4&-4, 4|9}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes00[i], op[i], false); + } + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes01[i], op[i], true); } - printf("--------------------json string---------------------\n"); + printf("--------------------json string- 0 op {1, 8, 2, 2, 3, 0, -4, 9}--------------------\n"); key = "k2"; - double eRes1[len] = {1.0, -8, 0, 0, 0, 0, 0, 9}; + double eRes10[len] = {1.0, -8, 0, 0, 0, 0, 0, 9}; + double eRes11[len] = {1.0, 8, 0, DBL_MAX, DBL_MAX, 0, 0, 9}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i], false); + } + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes11[i], op[i], true); } - printf("---------------------json null--------------------\n"); + printf("---------------------json null- null op {1, 8, 2, 2, 3, 0, -4, 9}-------------------\n"); key = "k3"; - double eRes2[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX}; + double eRes20[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX}; + double eRes21[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, 0, DBL_MAX, DBL_MAX}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes20[i], op[i], false); + } + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes21[i], op[i], true); } - printf("---------------------json bool--------------------\n"); + printf("---------------------json bool- true op {1, 8, 2, 2, 3, 0, -4, 9}-------------------\n"); key = "k4"; - double eRes3[len] = {2.0, -7, 2, 0.5, 1, -1, 1&-4, 1|9}; + double eRes30[len] = {2.0, -7, 2, 0.5, 1, -1, 1&-4, 1|9}; + double eRes31[len] = {2.0, 7, 2, 2, 0, 0, 1&-4, 1|9}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes30[i], op[i], false); + } + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes31[i], op[i], true); } - printf("----------------------json double-------------------\n"); + printf("----------------------json double-- 5.44 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n"); key = "k5"; - double eRes4[len] = {6.44, -2.56, 10.88, 2.72, 2.44, -5.44, 5&-4, 5|9}; + double eRes40[len] = {6.44, -2.56, 10.88, 2.72, 2.44, -5.44, 5&-4, 5|9}; + double eRes41[len] = {6.44, 2.56, 10.88, 0.3676470588235294, 3, 0, 5&-4, 5|9}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes40[i], op[i], false); + } + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes41[i], op[i], true); } - printf("---------------------json not exist--------------------\n"); + printf("----------------------json int-- -10 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n"); + + key = "k6"; + double eRes50[len] = {-9, -18, -20, -5, 2, 10, -10&-4, -10|9}; + double eRes51[len] = {-9, 18, -20, -0.2, -7, 0, -10&-4, -10|9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes50[i], op[i], false); + } + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes51[i], op[i], true); + } + + printf("----------------------json double-- -9.8 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n"); + + key = "k7"; + double eRes60[len] = {-8.8, -17.8, -19.6, -4.9, -9%3, 9.8, -9&-4, -9|9}; + double eRes61[len] = {-8.8, 17.8, -19.6, -0.2040816326530612, 3%-9, 0, -9&-4, -9|9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes60[i], op[i], false); + } + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes61[i], op[i], true); + } + + printf("----------------------json bool-- 0 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n"); + + key = "k8"; + double eRes70[len] = {1.0, -8, 0, 0, 0, 0, 0, 9}; + double eRes71[len] = {1.0, 8, 0, DBL_MAX, DBL_MAX, 0, 0, 9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes70[i], op[i], false); + } + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes70[i], op[i], true); + } + + printf("----------------------json string-- 8 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n"); + + key = "k9"; + double eRes80[len] = {9, 0, 16, 4, 8%3, -8, 8&-4, 8|9}; + double eRes81[len] = {9, 0, 16, 0.25, 3%8, 0, 8&-4, 8|9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes80[i], op[i], false); + } + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes81[i], op[i], true); + } + + printf("---------------------json not exist-- NULL op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n"); key = "k10"; - double eRes5[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX}; + double eRes90[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX}; + double eRes91[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, 0, DBL_MAX, DBL_MAX}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes90[i], op[i], false); + } + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes91[i], op[i], true); } taosArrayDestroy(tags); @@ -1203,11 +1293,11 @@ TEST(columnTest, json_column_logic_op) { char *key = "k1"; bool eRes[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i], false); } for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes[i], op[i], false); taosMemoryFree(rightData); } @@ -1216,11 +1306,11 @@ TEST(columnTest, json_column_logic_op) { key = "k2"; bool eRes1[len+len1] = {false, false, true, true, false, false, false, true, false, true, false, true, true}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i], false); } for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes1[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes1[i], op[i], false); taosMemoryFree(rightData); } @@ -1229,11 +1319,11 @@ TEST(columnTest, json_column_logic_op) { key = "k3"; // (null is true) return NULL, so use DBL_MAX represent NULL double eRes2[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i], false); } for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes2[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes2[i], op[i], false); taosMemoryFree(rightData); } @@ -1242,11 +1332,11 @@ TEST(columnTest, json_column_logic_op) { key = "k4"; bool eRes3[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i], false); } for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes3[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes3[i], op[i], false); taosMemoryFree(rightData); } @@ -1255,11 +1345,11 @@ TEST(columnTest, json_column_logic_op) { key = "k5"; bool eRes4[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i], false); } for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes4[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes4[i], op[i], false); taosMemoryFree(rightData); } @@ -1268,11 +1358,11 @@ TEST(columnTest, json_column_logic_op) { key = "k6"; bool eRes5[len+len1] = {true, false, false, false, false, true, false, true, true, false, true, false, true}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i], false); } for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i], false); taosMemoryFree(rightData); } @@ -1281,11 +1371,11 @@ TEST(columnTest, json_column_logic_op) { key = "k10"; // (NULL is true) return NULL, so use DBL_MAX represent NULL double eRes10[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i], false); } for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes10[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes10[i], op[i], false); taosMemoryFree(rightData); } diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 5c38d4afb9..6a855ebd4b 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -424,8 +424,8 @@ class TDTestCase: # test top/bottom with group by json tag tdSql.query("select top(dataint,2),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") tdSql.checkRows(11) - tdSql.checkData(0, 0, 24) tdSql.checkData(0, 1, None) + tdSql.checkData(2, 0, 4) tdSql.checkData(3, 0, 3) tdSql.checkData(3, 1, "false") tdSql.checkData(10, 0, 23) From 0633c25a2219b4616b9aa1be6300be95e84b3e0d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 5 Jun 2022 09:02:25 +0000 Subject: [PATCH 45/63] refact --- source/dnode/vnode/src/inc/tsdb.h | 151 +++++------------------ source/dnode/vnode/src/tsdb/tsdbCommit.c | 14 +-- source/dnode/vnode/src/tsdb/tsdbFile.c | 95 ++++++++++++++ 3 files changed, 133 insertions(+), 127 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 94e0a8cd07..42d56eb3e3 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -58,8 +58,31 @@ void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow); int32_t tsdbBegin2(STsdb *pTsdb); int32_t tsdbCommit2(STsdb *pTsdb); +// tsdbFile.c ============================================================================================== +typedef int32_t TSDB_FILE_T; +typedef struct SDFInfo SDFInfo; +typedef struct SDFile SDFile; +typedef struct SDFileSet SDFileSet; + +void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype); +void tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile); +int tsdbOpenDFile(SDFile *pDFile, int flags); +void tsdbCloseDFile(SDFile *pDFile); +int64_t tsdbSeekDFile(SDFile *pDFile, int64_t offset, int whence); +int64_t tsdbWriteDFile(SDFile *pDFile, void *buf, int64_t nbyte); +void tsdbUpdateDFileMagic(SDFile *pDFile, void *pCksm); +int tsdbAppendDFile(SDFile *pDFile, void *buf, int64_t nbyte, int64_t *offset); +int tsdbRemoveDFile(SDFile *pDFile); +int64_t tsdbReadDFile(SDFile *pDFile, void *buf, int64_t nbyte); +int tsdbCopyDFile(SDFile *pSrc, SDFile *pDest); +int tsdbEncodeSDFile(void **buf, SDFile *pDFile); +void *tsdbDecodeSDFile(STsdb *pRepo, void *buf, SDFile *pDFile); +int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T fType); +int tsdbUpdateDFileHeader(SDFile *pDFile); +int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo); +int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype, uint32_t *version); + // tsdbMemTable ================ -typedef struct STsdbRow STsdbRow; typedef struct STbData STbData; typedef struct STsdbMemTable STsdbMemTable; typedef struct SMergeInfo SMergeInfo; @@ -75,10 +98,6 @@ int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, SSkipListIterator *pIte // tsdbFS ================ typedef struct STsdbFS STsdbFS; -// tsdbSma ================ -typedef struct SSmaEnv SSmaEnv; -typedef struct SSmaEnvs SSmaEnvs; - // structs typedef struct { int minFid; @@ -113,7 +132,7 @@ struct STable { #define TABLE_TID(t) (t)->tid #define TABLE_UID(t) (t)->uid -int tsdbPrepareCommit(STsdb *pTsdb); +// int tsdbPrepareCommit(STsdb *pTsdb); typedef enum { TSDB_FILE_HEAD = 0, // .head TSDB_FILE_DATA, // .data @@ -124,7 +143,7 @@ typedef enum { TSDB_FILE_META, // meta } E_TSDB_FILE_T; -typedef struct { +struct SDFInfo { uint32_t magic; uint32_t fver; uint32_t len; @@ -133,22 +152,22 @@ typedef struct { uint32_t offset; uint64_t size; uint64_t tombSize; -} SDFInfo; +}; -typedef struct { +struct SDFile { SDFInfo info; STfsFile f; TdFilePtr pFile; uint8_t state; -} SDFile; +}; -typedef struct { +struct SDFileSet { int fid; int8_t state; // -128~127 uint8_t ver; // 0~255, DFileSet version uint16_t reserve; SDFile files[TSDB_FILE_MAX]; -} SDFileSet; +}; struct STbData { tb_uid_t uid; @@ -521,7 +540,6 @@ static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) { #define TSDB_FILE_STATE_OK 0 #define TSDB_FILE_STATE_BAD 1 -#define TSDB_FILE_INFO(tf) (&((tf)->info)) #define TSDB_FILE_F(tf) (&((tf)->f)) #define TSDB_FILE_PFILE(tf) ((tf)->pFile) #define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname) @@ -539,7 +557,6 @@ static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) { #define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK) #define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD) -typedef int32_t TSDB_FILE_T; typedef enum { TSDB_FS_VER_0 = 0, TSDB_FS_VER_MAX, @@ -560,112 +577,6 @@ static FORCE_INLINE uint32_t tsdbGetDFSVersion(TSDB_FILE_T fType) { // latest v } } -void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype); -void tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile); -int tsdbEncodeSDFile(void **buf, SDFile *pDFile); -void *tsdbDecodeSDFile(STsdb *pRepo, void *buf, SDFile *pDFile); -int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T fType); -int tsdbUpdateDFileHeader(SDFile *pDFile); -int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo); -int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype, uint32_t *version); - -static FORCE_INLINE void tsdbSetDFileInfo(SDFile *pDFile, SDFInfo *pInfo) { pDFile->info = *pInfo; } - -static FORCE_INLINE int tsdbOpenDFile(SDFile *pDFile, int flags) { - ASSERT(!TSDB_FILE_OPENED(pDFile)); - - pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags); - if (pDFile->pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return 0; -} - -static FORCE_INLINE void tsdbCloseDFile(SDFile *pDFile) { - if (TSDB_FILE_OPENED(pDFile)) { - taosCloseFile(&pDFile->pFile); - TSDB_FILE_SET_CLOSED(pDFile); - } -} - -static FORCE_INLINE int64_t tsdbSeekDFile(SDFile *pDFile, int64_t offset, int whence) { - // ASSERT(TSDB_FILE_OPENED(pDFile)); - - int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence); - if (loffset < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return loffset; -} - -static FORCE_INLINE int64_t tsdbWriteDFile(SDFile *pDFile, void *buf, int64_t nbyte) { - ASSERT(TSDB_FILE_OPENED(pDFile)); - - int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte); - if (nwrite < nbyte) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return nwrite; -} - -static FORCE_INLINE void tsdbUpdateDFileMagic(SDFile *pDFile, void *pCksm) { - pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t *)(pCksm), sizeof(TSCKSUM)); -} - -static FORCE_INLINE int tsdbAppendDFile(SDFile *pDFile, void *buf, int64_t nbyte, int64_t *offset) { - ASSERT(TSDB_FILE_OPENED(pDFile)); - - int64_t toffset; - - if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) { - return -1; - } - - ASSERT(pDFile->info.size == toffset); - - if (offset) { - *offset = toffset; - } - - if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) { - return -1; - } - - pDFile->info.size += nbyte; - - return (int)nbyte; -} - -static FORCE_INLINE int tsdbRemoveDFile(SDFile *pDFile) { return tfsRemoveFile(TSDB_FILE_F(pDFile)); } - -static FORCE_INLINE int64_t tsdbReadDFile(SDFile *pDFile, void *buf, int64_t nbyte) { - ASSERT(TSDB_FILE_OPENED(pDFile)); - - int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte); - if (nread < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return nread; -} - -static FORCE_INLINE int tsdbCopyDFile(SDFile *pSrc, SDFile *pDest) { - if (tfsCopyFile(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - tsdbSetDFileInfo(pDest, TSDB_FILE_INFO(pSrc)); - return 0; -} - // =============== SDFileSet typedef struct { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 3013f6deef..64e2b917ae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -218,15 +218,15 @@ static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) { return 0; } -int tsdbPrepareCommit(STsdb *pTsdb) { - if (pTsdb->mem == NULL) return 0; +// int tsdbPrepareCommit(STsdb *pTsdb) { +// if (pTsdb->mem == NULL) return 0; - ASSERT(pTsdb->imem == NULL); +// ASSERT(pTsdb->imem == NULL); - pTsdb->imem = pTsdb->mem; - pTsdb->mem = NULL; - return 0; -} +// pTsdb->imem = pTsdb->mem; +// pTsdb->mem = NULL; +// return 0; +// } void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) { STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index e7c8c7ccd9..1e9c256d9c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -447,4 +447,99 @@ static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, c snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s-ver%" PRIu32, vid, TSDB_FNAME_SUFFIX[ftype], ver); } } +} + +int tsdbOpenDFile(SDFile *pDFile, int flags) { + ASSERT(!TSDB_FILE_OPENED(pDFile)); + + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags); + if (pDFile->pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return 0; +} + +void tsdbCloseDFile(SDFile *pDFile) { + if (TSDB_FILE_OPENED(pDFile)) { + taosCloseFile(&pDFile->pFile); + TSDB_FILE_SET_CLOSED(pDFile); + } +} + +int64_t tsdbSeekDFile(SDFile *pDFile, int64_t offset, int whence) { + // ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence); + if (loffset < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return loffset; +} + +int64_t tsdbWriteDFile(SDFile *pDFile, void *buf, int64_t nbyte) { + ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte); + if (nwrite < nbyte) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nwrite; +} + +void tsdbUpdateDFileMagic(SDFile *pDFile, void *pCksm) { + pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t *)(pCksm), sizeof(TSCKSUM)); +} + +int tsdbAppendDFile(SDFile *pDFile, void *buf, int64_t nbyte, int64_t *offset) { + ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t toffset; + + if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) { + return -1; + } + + ASSERT(pDFile->info.size == toffset); + + if (offset) { + *offset = toffset; + } + + if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) { + return -1; + } + + pDFile->info.size += nbyte; + + return (int)nbyte; +} + +int tsdbRemoveDFile(SDFile *pDFile) { return tfsRemoveFile(TSDB_FILE_F(pDFile)); } + +int64_t tsdbReadDFile(SDFile *pDFile, void *buf, int64_t nbyte) { + ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte); + if (nread < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nread; +} + +int tsdbCopyDFile(SDFile *pSrc, SDFile *pDest) { + if (tfsCopyFile(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + pDest->info = pSrc->info; + return 0; } \ No newline at end of file From ac09ef04a96c27a9ac1f7d4779dfe8f8b42911ad Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sun, 5 Jun 2022 17:02:43 +0800 Subject: [PATCH 46/63] fix(os): add win32 udf case --- source/libs/function/src/udfd.c | 8 +++ source/libs/function/test/udf1.c | 6 +- source/libs/function/test/udf2.c | 10 ++-- source/os/src/osFile.c | 6 ++ source/os/src/osSysinfo.c | 13 ++++- source/os/src/osTimezone.c | 81 +++++++++++++-------------- tests/pytest/util/dnodes.py | 17 +++--- tests/system-test/0-others/udfTest.py | 63 ++++++++++++--------- tests/system-test/fulltest.bat | 3 +- 9 files changed, 118 insertions(+), 89 deletions(-) diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 83dcb6d7f0..02c485fa83 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -401,9 +401,17 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { udf->bufSize = pFuncInfo->bufSize; char path[PATH_MAX] = {0}; + #ifdef WINDOWS + snprintf(path, sizeof(path), "%s%s.dll", TD_TMP_DIR_PATH, pFuncInfo->name); + #else snprintf(path, sizeof(path), "%s/lib%s.so", TD_TMP_DIR_PATH, pFuncInfo->name); + #endif TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); + if (file == NULL) { + fnError("udfd write udf shared library: %s failed, error: %d %s", path, errno, strerror(errno)); + msgInfo->code = TSDB_CODE_FILE_CORRUPTED; + } int64_t count = taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); if (count != pFuncInfo->codeSize) { fnError("udfd write udf shared library failed"); diff --git a/source/libs/function/test/udf1.c b/source/libs/function/test/udf1.c index e58c9cc00a..9443d5cb94 100644 --- a/source/libs/function/test/udf1.c +++ b/source/libs/function/test/udf1.c @@ -9,15 +9,15 @@ #undef free #define free free -int32_t udf1_init() { +DLL_EXPORT int32_t udf1_init() { return 0; } -int32_t udf1_destroy() { +DLL_EXPORT int32_t udf1_destroy() { return 0; } -int32_t udf1(SUdfDataBlock* block, SUdfColumn *resultCol) { +DLL_EXPORT int32_t udf1(SUdfDataBlock* block, SUdfColumn *resultCol) { SUdfColumnMeta *meta = &resultCol->colMeta; meta->bytes = 4; meta->type = TSDB_DATA_TYPE_INT; diff --git a/source/libs/function/test/udf2.c b/source/libs/function/test/udf2.c index 49d681f5eb..1c270f5cf4 100644 --- a/source/libs/function/test/udf2.c +++ b/source/libs/function/test/udf2.c @@ -9,22 +9,22 @@ #undef free #define free free -int32_t udf2_init() { +DLL_EXPORT int32_t udf2_init() { return 0; } -int32_t udf2_destroy() { +DLL_EXPORT int32_t udf2_destroy() { return 0; } -int32_t udf2_start(SUdfInterBuf *buf) { +DLL_EXPORT int32_t udf2_start(SUdfInterBuf *buf) { *(int64_t*)(buf->buf) = 0; buf->bufLen = sizeof(double); buf->numOfResult = 0; return 0; } -int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { +DLL_EXPORT int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { double sumSquares = *(double*)interBuf->buf; int8_t numNotNull = 0; for (int32_t i = 0; i < block->numOfCols; ++i) { @@ -71,7 +71,7 @@ int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInte return 0; } -int32_t udf2_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) { +DLL_EXPORT int32_t udf2_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) { if (buf->numOfResult == 0) { resultData->numOfResult = 0; return 0; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index c75cca79f6..32fed66cd4 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -399,6 +399,9 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) } int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { + if (pFile == NULL) { + return 0; + } #if FILE_WITH_LOCK taosThreadRwlockWrlock(&(pFile->rwlock)); #endif @@ -430,6 +433,9 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { } int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { + if (pFile == NULL) { + return 0; + } #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 4d7b15401c..ace870853f 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -276,7 +276,8 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { #ifdef WINDOWS - assert(0); + snprintf(releaseName, maxLen, "Windows"); + return 0; #elif defined(_TD_DARWIN_64) char *line = NULL; size_t size = 0; @@ -332,7 +333,15 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { #ifdef WINDOWS - assert(0); + char value[100]; + DWORD bufferSize = sizeof(value); + RegGetValue(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", "ProcessorNameString", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); + tstrncpy(cpuModel, value, maxLen); + SYSTEM_INFO si; + memset(&si,0,sizeof(SYSTEM_INFO)); + GetSystemInfo(&si); + *numOfCores = si.dwNumberOfProcessors; + return 0; #elif defined(_TD_DARWIN_64) char *line = NULL; size_t size = 0; diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index dc9527c2f2..724f81c66c 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -767,32 +767,36 @@ void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8 keyValue[4] = (keyValue[4] == '+' ? '-' : '+'); keyValue[10] = 0; sprintf(winStr, "TZ=%s:00", &(keyValue[1])); + *tsTimezone = taosStr2Int32(&keyValue[4], NULL, 10); } break; } } - char *p = strchr(inTimezoneStr, '+'); - if (p == NULL) p = strchr(inTimezoneStr, '-'); - if (p == NULL) { - sprintf(winStr, "TZ=UTC+00:00:00"); - } else { - sprintf(winStr, "TZ=UTC%c%c%c:%c%c:00", (p[0] == '+' ? '-' : '+'), p[1], p[2], p[3], p[4]); + if (winStr[0] == 0) { + char *p = strchr(inTimezoneStr, '+'); + if (p == NULL) p = strchr(inTimezoneStr, '-'); + if (p != NULL) { + char *pp = strchr(inTimezoneStr, '('); + char *ppp = strchr(inTimezoneStr, ','); + int indexStr; + if (pp == NULL || ppp == NULL) { + indexStr = sprintf(winStr, "TZ=UTC"); + } else { + memcpy(winStr, "TZ=", 3); + pp++; + memcpy(&winStr[3], pp, ppp - pp); + indexStr = ppp - pp + 3; + } + sprintf(&winStr[indexStr], "%c%c%c:%c%c:00", (p[0] == '+'? '-' : '+'), p[1], p[2], p[3], p[4]); + *tsTimezone = taosStr2Int32(p, NULL, 10); + } else { + *tsTimezone = 0; + } } _putenv(winStr); _tzset(); -#ifdef _MSC_VER -#if _MSC_VER >= 1900 - int64_t timezone = _timezone; - int32_t daylight = _daylight; - char **tzname = _tzname; -#endif -#endif - - int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); - *tsTimezone = tz; - tz += daylight; - sprintf(outTimezoneStr, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); - *outDaylight = daylight; + strcpy(outTimezoneStr, inTimezoneStr); + *outDaylight = 0; #elif defined(_TD_DARWIN_64) @@ -822,34 +826,27 @@ void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8 void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { #ifdef WINDOWS char value[100]; + char keyPath[100]; DWORD bufferSize = sizeof(value); - char *buf = getenv("TZ"); - if (buf == NULL || strlen(buf) == 0) { - RegGetValue(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\TimeZoneInformation", "TimeZoneKeyName", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); - strcpy(outTimezoneStr, "not configured"); - if (bufferSize > 0) { - for (size_t i = 0; i < 139; i++) { - if (strcmp(win_tz[i][0],value) == 0) { - strcpy(outTimezoneStr, win_tz[i][1]); - break; + RegGetValue(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\TimeZoneInformation", "TimeZoneKeyName", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); + strcpy(outTimezoneStr, "not configured"); + *tsTimezone = 0; + if (bufferSize > 0) { + for (size_t i = 0; i < 139; i++) { + if (strcmp(win_tz[i][0],value) == 0) { + strcpy(outTimezoneStr, win_tz[i][1]); + bufferSize = sizeof(value); + sprintf(keyPath, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Time Zones\\%s",value); + RegGetValue(HKEY_LOCAL_MACHINE, keyPath, "Display", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); + if (bufferSize > 0) { + // value[4] = (value[4] == '+' ? '-' : '+'); + sprintf(outTimezoneStr, "%s (UTC, %c%c%c%c%c)", outTimezoneStr, value[4], value[5], value[6], value[8], value[9]); + *tsTimezone = taosStr2Int32(&value[4], NULL, 10); } + break; } } - } else { - strcpy(outTimezoneStr, buf); } -#ifdef _MSC_VER -#if _MSC_VER >= 1900 - // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 - int64_t timezone = _timezone; - int32_t daylight = _daylight; - char **tzname = _tzname; -#endif -#endif - int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); - *tsTimezone = tz; - tz += daylight; - sprintf(outTimezoneStr, "%s (%s, %s%02d00)", outTimezoneStr, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); #elif defined(_TD_DARWIN_64) char buf[4096] = {0}; char *tz = NULL; diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 7eeddfd5c1..07518219ce 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -145,6 +145,12 @@ class TDDnode: def init(self, path, remoteIP = ""): self.path = path self.remoteIP = remoteIP + if (not self.remoteIP == ""): + try: + self.config = eval(self.remoteIP) + self.remote_conn = Connection(host=self.config["host"], port=self.config["port"], user=self.config["user"], connect_kwargs={'password':self.config["password"]}) + except Exception as r: + print(r) def setTestCluster(self, value): self.testCluster = value @@ -169,13 +175,6 @@ class TDDnode: self.cfgDict.update({option: value}) def remoteExec(self, updateCfgDict, execCmd): - try: - config = eval(self.remoteIP) - remote_conn = Connection(host=config["host"], port=config["port"], user=config["user"], connect_kwargs={'password':config["password"]}) - remote_top_dir = config["path"] - except Exception as r: - remote_conn = Connection(host=self.remoteIP, port=22, user='root', connect_kwargs={'password':'123456'}) - remote_top_dir = '~/test' valgrindStr = '' if (self.valgrind==1): valgrindStr = '-g' @@ -188,8 +187,8 @@ class TDDnode: del remoteCfgDict["cfgDir"] remoteCfgDictStr = base64.b64encode(json.dumps(remoteCfgDict).encode()).decode() execCmdStr = base64.b64encode(execCmd.encode()).decode() - with remote_conn.cd((remote_top_dir+sys.path[0].replace(self.path, '')).replace('\\','/')): - remote_conn.run("python3 ./test.py %s -d %s -e %s"%(valgrindStr,remoteCfgDictStr,execCmdStr)) + with self.remote_conn.cd((self.config["path"]+sys.path[0].replace(self.path, '')).replace('\\','/')): + self.remote_conn.run("python3 ./test.py %s -d %s -e %s"%(valgrindStr,remoteCfgDictStr,execCmdStr)) def deploy(self, *updatecfgDict): self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index) diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index 46d0a69688..57ae1c1490 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -3,6 +3,7 @@ import taos import sys import time import os +import platform from util.log import * from util.sql import * @@ -25,7 +26,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] @@ -41,11 +42,19 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] print(projPath) - libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") - libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") - os.system("mkdir /tmp/udf/") - os.system("cp %s /tmp/udf/ "%libudf1.replace("\n" ,"")) - os.system("cp %s /tmp/udf/ "%libudf2.replace("\n" ,"")) + if platform.system().lower() == 'windows': + self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + if (not tdDnodes.dnodes[0].remoteIP == ""): + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\") + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\") + self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so') + self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so') + else: + self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf1 = self.libudf1.replace('\r','').replace('\n','') + self.libudf2 = self.libudf2.replace('\r','').replace('\n','') def prepare_data(self): @@ -136,11 +145,11 @@ class TDTestCase: for i in range(5): # create scalar functions - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;") + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) # create aggregate functions - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;") + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) functions = tdSql.getResult("show functions") function_nums = len(functions) @@ -161,11 +170,11 @@ class TDTestCase: tdLog.info("drop two udf functions success ") # create scalar functions - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;") + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) # create aggregate functions - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;") + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) functions = tdSql.getResult("show functions") function_nums = len(functions) @@ -533,8 +542,8 @@ class TDTestCase: tdSql.query("drop function udf2 ") # create function without buffer - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int") - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double") + tdSql.execute("create function udf1 as '%s' outputtype int"%self.libudf1) + tdSql.execute("create aggregate function udf2 as '%s' outputtype double"%self.libudf2) udf1_sqls ,udf2_sqls = self.try_query_sql() for scalar_sql in udf1_sqls: @@ -549,8 +558,8 @@ class TDTestCase: tdSql.query("drop function udf2 ") # create function without buffer - tdSql.execute("create aggregate function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") - tdSql.execute("create function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8") + tdSql.execute("create aggregate function udf1 as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute("create function udf2 as '%s' outputtype double bufSize 8"%self.libudf2) udf1_sqls ,udf2_sqls = self.try_query_sql() for scalar_sql in udf1_sqls: @@ -558,8 +567,8 @@ class TDTestCase: for aggregate_sql in udf2_sqls: tdSql.error(aggregate_sql) - tdSql.execute(" create function db as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") - tdSql.execute(" create aggregate function test as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") + tdSql.execute(" create function db as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute(" create aggregate function test as '%s' outputtype int bufSize 8 "%self.libudf1) tdSql.error(" select db(c1) from stb1 ") tdSql.error(" select db(c1,c6), db(c6) from stb1 ") tdSql.error(" select db(num1,num2), db(num1) from tb ") @@ -607,17 +616,17 @@ class TDTestCase: tdLog.info(" create function name is not build_in functions ") tdSql.execute(" drop function udf1 ") tdSql.execute(" drop function udf2 ") - tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8") - tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8") - tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function tbname as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function function as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function stable as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function union as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function 123 as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function 123db as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function mnode as '/tmp/udf/libudf2.so' outputtype double bufSize 8") + tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function tbname as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function function as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function stable as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function union as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function 123 as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function 123db as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function mnode as '%s' outputtype double bufSize 8"%self.libudf2) def restart_taosd_query_udf(self): diff --git a/tests/system-test/fulltest.bat b/tests/system-test/fulltest.bat index 871c93c982..f4da0aabb3 100644 --- a/tests/system-test/fulltest.bat +++ b/tests/system-test/fulltest.bat @@ -1,4 +1,5 @@ python3 .\test.py -f 0-others\taosShell.py python3 .\test.py -f 0-others\taosShellError.py -python3 .\test.py -f 0-others\taosShellNetChk.py \ No newline at end of file +python3 .\test.py -f 0-others\taosShellNetChk.py +python3 .\test.py -f 0-others\udfTest.py \ No newline at end of file From 40a396563bd7feb9fc23a339480f6fb771e5a668 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 5 Jun 2022 09:23:42 +0000 Subject: [PATCH 47/63] more refact --- source/dnode/vnode/src/inc/tsdb.h | 97 +++++----------------- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFile.c | 38 +++++++++ source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 6 +- 4 files changed, 63 insertions(+), 80 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 42d56eb3e3..d2429391ab 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -64,6 +64,7 @@ typedef struct SDFInfo SDFInfo; typedef struct SDFile SDFile; typedef struct SDFileSet SDFileSet; +// SDFile void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype); void tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile); int tsdbOpenDFile(SDFile *pDFile, int flags); @@ -82,6 +83,26 @@ int tsdbUpdateDFileHeader(SDFile *pDFile); int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo); int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype, uint32_t *version); +// SDFileSet +void tsdbInitDFileSet(STsdb *pRepo, SDFileSet *pSet, SDiskID did, int fid, uint32_t ver); +void tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet); +int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet); +void *tsdbDecodeDFileSet(STsdb *pRepo, void *buf, SDFileSet *pSet); +int tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet); +void *tsdbDecodeDFileSetEx(void *buf, SDFileSet *pSet); +int tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to); +int tsdbCreateDFileSet(STsdb *pRepo, SDFileSet *pSet, bool updateHeader); +int tsdbUpdateDFileSetHeader(SDFileSet *pSet); +int tsdbScanAndTryFixDFileSet(STsdb *pRepo, SDFileSet *pSet); +void tsdbCloseDFileSet(SDFileSet *pSet); +int tsdbOpenDFileSet(SDFileSet *pSet, int flags); +void tsdbRemoveDFileSet(SDFileSet *pSet); +int tsdbCopyDFileSet(SDFileSet *pSrc, SDFileSet *pDest); +void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY *minKey, TSKEY *maxKey); + +// tsdbFS.c ============================================================================================== +typedef struct STsdbFS STsdbFS; + // tsdbMemTable ================ typedef struct STbData STbData; typedef struct STsdbMemTable STsdbMemTable; @@ -93,11 +114,6 @@ void tsdbMemTableDestroy(STsdbMemTable *pMemTable); int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo); -// tsdbCommit ================ - -// tsdbFS ================ -typedef struct STsdbFS STsdbFS; - // structs typedef struct { int minFid; @@ -313,8 +329,6 @@ typedef struct { uint64_t aggrOffset : 63; TSDBKEY minKey; TSDBKEY maxKey; - // TSKEY keyFirst; - // TSKEY keyLast; } SBlock; static FORCE_INLINE bool tsdbIsSupBlock(SBlock *pBlock) { return pBlock->numOfSubBlocks == 1; } @@ -345,26 +359,6 @@ typedef struct { int64_t min; } SAggrBlkCol; -// Code here just for back-ward compatibility -static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) { -#ifdef TD_REFACTOR_3 - pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1); - pBlockCol->offsetH = (uint8_t)(offset >> 24); -#else - pBlockCol->offset = offset; -#endif -} - -static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) { -#ifdef TD_REFACTOR_3 - uint32_t offset1 = pBlockCol->offset; - uint32_t offset2 = pBlockCol->offsetH; - return (offset1 | (offset2 << 24)); -#else - return pBlockCol->offset; -#endif -} - typedef struct { int32_t delimiter; // For recovery usage int32_t numOfCols; // For recovery usage @@ -613,55 +607,6 @@ typedef struct { } \ } while (0); -void tsdbInitDFileSet(STsdb *pRepo, SDFileSet *pSet, SDiskID did, int fid, uint32_t ver); -void tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet); -int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet); -void *tsdbDecodeDFileSet(STsdb *pRepo, void *buf, SDFileSet *pSet); -int tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet); -void *tsdbDecodeDFileSetEx(void *buf, SDFileSet *pSet); -int tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to); -int tsdbCreateDFileSet(STsdb *pRepo, SDFileSet *pSet, bool updateHeader); -int tsdbUpdateDFileSetHeader(SDFileSet *pSet); -int tsdbScanAndTryFixDFileSet(STsdb *pRepo, SDFileSet *pSet); - -static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet *pSet) { - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype)); - } -} - -static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet *pSet, int flags) { - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) { - tsdbCloseDFileSet(pSet); - return -1; - } - } - return 0; -} - -static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet *pSet) { - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - (void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype)); - } -} - -static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet *pSrc, SDFileSet *pDest) { - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) { - tsdbRemoveDFileSet(pDest); - return -1; - } - } - - return 0; -} - -static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY *minKey, TSKEY *maxKey) { - *minKey = fid * days * tsTickPerMin[precision]; - *maxKey = *minKey + days * tsTickPerMin[precision] - 1; -} - static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet *pSet) { for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { if (TSDB_FILE_IS_BAD(TSDB_DFILE_IN_SET(pSet, ftype))) { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 64e2b917ae..10b890262e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1175,7 +1175,7 @@ static int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFi tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM))); if (ncol != 0) { - tsdbSetBlockColOffset(pBlockCol, toffset); + pBlockCol->offset = toffset; pBlockCol->len = flen; // data + bitmaps pBlockCol->blen = tBitmapsLen; ++tcol; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 1e9c256d9c..a88cb8a353 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -542,4 +542,42 @@ int tsdbCopyDFile(SDFile *pSrc, SDFile *pDest) { pDest->info = pSrc->info; return 0; +} + +void tsdbCloseDFileSet(SDFileSet *pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype)); + } +} + +int tsdbOpenDFileSet(SDFileSet *pSet, int flags) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) { + tsdbCloseDFileSet(pSet); + return -1; + } + } + return 0; +} + +void tsdbRemoveDFileSet(SDFileSet *pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + (void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype)); + } +} + +int tsdbCopyDFileSet(SDFileSet *pSrc, SDFileSet *pDest) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) { + tsdbRemoveDFileSet(pDest); + return -1; + } + } + + return 0; +} + +void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY *minKey, TSKEY *maxKey) { + *minKey = fid * days * tsTickPerMin[precision]; + *maxKey = *minKey + days * tsTickPerMin[precision] - 1; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index e333f0566b..a26436156d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -726,7 +726,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat if (dcol != 0) { pBlockCol = &(pBlockData->cols[ccol]); tcolId = pBlockCol->colId; - toffset = tsdbGetBlockColOffset(pBlockCol); + toffset = pBlockCol->offset; tlen = pBlockCol->len; pDataCol->bitmap = pBlockCol->blen > 0 ? 1 : 0; } else { @@ -942,8 +942,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1; if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1; - int64_t offset = pBlock->offset + tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer) + - tsdbGetBlockColOffset(pBlockCol); + int64_t offset = + pBlock->offset + tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer) + pBlockCol->offset; if (tsdbSeekDFile(pDFile, offset, SEEK_SET) < 0) { tsdbError("vgId:%d, failed to load block column data while seek file %s to offset %" PRId64 " since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, tstrerror(terrno)); From bbc2ba9fefa91b6a437a2f8e841710c66e9e230a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 5 Jun 2022 09:29:18 +0000 Subject: [PATCH 48/63] refact TSDB --- source/dnode/vnode/src/inc/tsdb.h | 90 +++++++++------------------- source/dnode/vnode/src/tsdb/tsdbFS.c | 34 ++++++++--- 2 files changed, 54 insertions(+), 70 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index d2429391ab..ca80fe394a 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -101,7 +101,28 @@ int tsdbCopyDFileSet(SDFileSet *pSrc, SDFileSet *pDest); void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY *minKey, TSKEY *maxKey); // tsdbFS.c ============================================================================================== -typedef struct STsdbFS STsdbFS; +typedef struct STsdbFS STsdbFS; +typedef struct SFSIter SFSIter; +typedef struct STsdbFSMeta STsdbFSMeta; + +STsdbFS *tsdbNewFS(const STsdbKeepCfg *pCfg); +void *tsdbFreeFS(STsdbFS *pfs); +int tsdbOpenFS(STsdb *pRepo); +void tsdbCloseFS(STsdb *pRepo); +void tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd); +int tsdbEndFSTxn(STsdb *pRepo); +int tsdbEndFSTxnWithError(STsdbFS *pfs); +void tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta); +// void tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile); +int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet); + +void tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction); +void tsdbFSIterSeek(SFSIter *pIter, int fid); +SDFileSet *tsdbFSIterNext(SFSIter *pIter); +int tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta); +int tsdbRLockFS(STsdbFS *pFs); +int tsdbWLockFS(STsdbFS *pFs); +int tsdbUnLockFS(STsdbFS *pFs); // tsdbMemTable ================ typedef struct STbData STbData; @@ -208,11 +229,11 @@ struct STsdbMemTable { SHashObj *pHashIdx; }; -typedef struct { +struct STsdbFSMeta { uint32_t version; // Commit version from 0 to increase int64_t totalPoints; // total points int64_t totalStorage; // Uncompressed total storage -} STsdbFSMeta; +}; // ================== typedef struct { @@ -573,21 +594,7 @@ static FORCE_INLINE uint32_t tsdbGetDFSVersion(TSDB_FILE_T fType) { // latest v // =============== SDFileSet -typedef struct { - int fid; - int8_t state; - uint8_t ver; - uint16_t reserve; -#if 0 - SDFInfo info; -#endif - STfsFile f; - TdFilePtr pFile; - -} SSFile; // files split by days with fid - -#define TSDB_LATEST_FSET_VER 0 - +#define TSDB_LATEST_FSET_VER 0 #define TSDB_FSET_FID(s) ((s)->fid) #define TSDB_FSET_STATE(s) ((s)->state) #define TSDB_FSET_VER(s) ((s)->ver) @@ -634,61 +641,18 @@ typedef struct { #define FS_VERSION(pfs) ((pfs)->cstatus->meta.version) #define FS_TXN_VERSION(pfs) ((pfs)->nstatus->meta.version) -typedef struct { +struct SFSIter { int direction; uint64_t version; // current FS version STsdbFS *pfs; int index; // used to position next fset when version the same int fid; // used to seek when version is changed SDFileSet *pSet; -} SFSIter; +}; #define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC #define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC -STsdbFS *tsdbNewFS(const STsdbKeepCfg *pCfg); -void *tsdbFreeFS(STsdbFS *pfs); -int tsdbOpenFS(STsdb *pRepo); -void tsdbCloseFS(STsdb *pRepo); -void tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd); -int tsdbEndFSTxn(STsdb *pRepo); -int tsdbEndFSTxnWithError(STsdbFS *pfs); -void tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta); -// void tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile); -int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet); - -void tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction); -void tsdbFSIterSeek(SFSIter *pIter, int fid); -SDFileSet *tsdbFSIterNext(SFSIter *pIter); -int tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta); - -static FORCE_INLINE int tsdbRLockFS(STsdbFS *pFs) { - int code = taosThreadRwlockRdlock(&(pFs->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE int tsdbWLockFS(STsdbFS *pFs) { - int code = taosThreadRwlockWrlock(&(pFs->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { - int code = taosThreadRwlockUnlock(&(pFs->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - struct TSDBROW { int64_t version; STSRow2 tsRow; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 19a79ac5c4..f1941a3bad 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -952,13 +952,6 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { } static int tsdbRestoreCurrent(STsdb *pRepo) { - // // Loop to recover mfile - // if (tsdbRestoreMeta(pRepo) < 0) { - // tsdbError("vgId:%d, failed to restore current since %s", REPO_ID(pRepo), tstrerror(terrno)); - // return -1; - // } - - // Loop to recover dfile set if (tsdbRestoreDFileSet(pRepo) < 0) { tsdbError("vgId:%d, failed to restore DFileSet since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; @@ -1041,3 +1034,30 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired) { tsdbCloseDFileSet(&fset); } } + +int tsdbRLockFS(STsdbFS *pFs) { + int code = taosThreadRwlockRdlock(&(pFs->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +int tsdbWLockFS(STsdbFS *pFs) { + int code = taosThreadRwlockWrlock(&(pFs->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +int tsdbUnLockFS(STsdbFS *pFs) { + int code = taosThreadRwlockUnlock(&(pFs->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} \ No newline at end of file From 020372cc7502a7e3c5e61e3c4252675b065c16bb Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 5 Jun 2022 18:26:16 +0800 Subject: [PATCH 49/63] feat: add sort/group logic for json --- source/common/src/tdatablock.c | 2 +- source/libs/scalar/inc/filterInt.h | 2 +- .../libs/scalar/test/scalar/scalarTests.cpp | 132 +++++++++++++++--- 3 files changed, 114 insertions(+), 22 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 055992db53..cb5fcfd6f9 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -109,7 +109,7 @@ int32_t getJsonValueLen(const char *data) { dataLen = DOUBLE_BYTES + CHAR_BYTES; } else if (*data == TSDB_DATA_TYPE_BOOL) { dataLen = CHAR_BYTES + CHAR_BYTES; - } else if (*data == TD_TAG_JSON) { // json string + } else if (*data & TD_TAG_JSON) { // json string dataLen = ((STag*)(data))->len; } else { ASSERT(0); diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index f4cd9dd1c1..54e873065b 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -350,7 +350,7 @@ struct SFilterInfo { extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right); extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr); - +extern OptrStr gOptrStr[]; #ifdef __cplusplus } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 9519c7d541..502ec17ead 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -42,6 +42,7 @@ #include "nodes.h" #include "tlog.h" #include "parUtil.h" +#include "filterInt.h" #define _DEBUG_PRINT_ 0 @@ -1106,16 +1107,16 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do }else if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || opType == OP_TYPE_DIV || opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){ - printf("1result:%f,except:%f\n", *((double *)colDataGetData(column, 0)), exceptValue); + printf("op:%s,1result:%f,except:%f\n", gOptrStr[opType].str, *((double *)colDataGetData(column, 0)), exceptValue); ASSERT_TRUE(fabs(*((double *)colDataGetData(column, 0)) - exceptValue) < 0.0001); }else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){ - printf("2result:%ld,except:%f\n", *((int64_t *)colDataGetData(column, 0)), exceptValue); + printf("op:%s,2result:%ld,except:%f\n", gOptrStr[opType].str, *((int64_t *)colDataGetData(column, 0)), exceptValue); ASSERT_EQ(*((int64_t *)colDataGetData(column, 0)), exceptValue); }else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL || opType == OP_TYPE_LOWER_THAN || opType == OP_TYPE_LOWER_EQUAL || opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL || opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE || opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH || opType == OP_TYPE_NMATCH){ - printf("3result:%d,except:%f\n", *((bool *)colDataGetData(column, 0)), exceptValue); + printf("op:%s,3result:%d,except:%f\n", gOptrStr[opType].str, *((bool *)colDataGetData(column, 0)), exceptValue); ASSERT_EQ(*((bool *)colDataGetData(column, 0)), exceptValue); } @@ -1188,7 +1189,7 @@ TEST(columnTest, json_column_arith_op) { printf("----------------------json double-- 5.44 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n"); key = "k5"; - double eRes40[len] = {6.44, -2.56, 10.88, 2.72, 2.44, -5.44, 5&-4, 5|9}; + double eRes40[len] = {6.44, -2.56, 10.88, 2.72, 2, -5.44, 5&-4, 5|9}; double eRes41[len] = {6.44, 2.56, 10.88, 0.3676470588235294, 3, 0, 5&-4, 5|9}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes40[i], op[i], false); @@ -1200,8 +1201,8 @@ TEST(columnTest, json_column_arith_op) { printf("----------------------json int-- -10 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n"); key = "k6"; - double eRes50[len] = {-9, -18, -20, -5, 2, 10, -10&-4, -10|9}; - double eRes51[len] = {-9, 18, -20, -0.2, -7, 0, -10&-4, -10|9}; + double eRes50[len] = {-9, -18, -20, -5, -10%3, 10, -10&-4, -10|9}; + double eRes51[len] = {-9, 18, -20, -0.2, 3%-10, 0, -10&-4, -10|9}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes50[i], op[i], false); } @@ -1230,7 +1231,7 @@ TEST(columnTest, json_column_arith_op) { makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes70[i], op[i], false); } for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes70[i], op[i], true); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes71[i], op[i], true); } printf("----------------------json string-- 8 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n"); @@ -1273,7 +1274,7 @@ void *prepareNchar(char* rightData){ TEST(columnTest, json_column_logic_op) { scltInitLogFile(); - char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":\"6.6hello\"}"; + char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":-10,\"k7\":-9.8,\"k8\":false,\"k9\":\"6.6hello\"}"; char rightv[256] = {0}; memcpy(rightv, rightvTmp, strlen(rightvTmp)); @@ -1281,6 +1282,7 @@ TEST(columnTest, json_column_logic_op) { STag* row = NULL; parseJsontoTagData(rightv, tags, &row, NULL); + const int32_t len0 = 6; const int32_t len = 9; const int32_t len1 = 4; EOperatorType op[len+len1] = {OP_TYPE_GREATER_THAN, OP_TYPE_GREATER_EQUAL, OP_TYPE_LOWER_THAN, OP_TYPE_LOWER_EQUAL, OP_TYPE_EQUAL, OP_TYPE_NOT_EQUAL, @@ -1289,93 +1291,183 @@ TEST(columnTest, json_column_logic_op) { int32_t input[len] = {1, 8, 2, 2, 3, 0, 0, 0, 0}; char *inputNchar[len1] = {"hell_", "hel%", "hell", "llll"}; - printf("--------------------json int---------------------\n"); + printf("--------------------json int---4 {1, 8, 2, 2, 3, 0, 0, 0, 0}------------------\n"); char *key = "k1"; bool eRes[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i], false); } + bool eRes_0[len0] = {false, true, true, true, false, true}; + for(int i = 0; i < len0; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_0[i], op[i], true); + } for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes[i], op[i], false); taosMemoryFree(rightData); } - printf("--------------------json string---------------------\n"); + printf("--------------------json string--0 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); key = "k2"; bool eRes1[len+len1] = {false, false, true, true, false, false, false, true, false, true, false, true, true}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i], false); } + bool eRes_1[len0] = {true, true, false, false, false, false}; + for(int i = 0; i < len0; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_1[i], op[i], true); + } + for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes1[i], op[i], false); taosMemoryFree(rightData); } - printf("--------------------json null---------------------\n"); + printf("--------------------json null---null {1, 8, 2, 2, 3, 0, 0, 0, 0}------------------\n"); key = "k3"; // (null is true) return NULL, so use DBL_MAX represent NULL double eRes2[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i], false); } + bool eRes_2[len0] = {false, false, false, false, false, false}; + for(int i = 0; i < len0; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_2[i], op[i], true); + } + for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes2[i], op[i], false); taosMemoryFree(rightData); } - printf("--------------------json bool---------------------\n"); + printf("--------------------json bool--1 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); key = "k4"; bool eRes3[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i], false); } + bool eRes_3[len0] = {false, true, false, false, false, true}; + for(int i = 0; i < len0; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_3[i], op[i], true); + } + for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes3[i], op[i], false); taosMemoryFree(rightData); } - printf("--------------------json double---------------------\n"); + printf("--------------------json double--5.44 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); key = "k5"; bool eRes4[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i], false); } + bool eRes_4[len0] = {false, true, true, true, false, true}; + for(int i = 0; i < len0; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_4[i], op[i], true); + } + for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes4[i], op[i], false); taosMemoryFree(rightData); } - printf("--------------------json double---------------------\n"); + printf("--------------------json int-- -10 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); key = "k6"; - bool eRes5[len+len1] = {true, false, false, false, false, true, false, true, true, false, true, false, true}; + bool eRes5[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, true}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i], false); } + bool eRes_5[len0] = {true, true, false, false, false, true}; + for(int i = 0; i < len0; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_5[i], op[i], true); + } + for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i], false); taosMemoryFree(rightData); } - printf("---------------------json not exist--------------------\n"); + printf("--------------------json double-- -9.8 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); - key = "k10"; // (NULL is true) return NULL, so use DBL_MAX represent NULL - double eRes10[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false}; + key = "k7"; + bool eRes6[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i], false); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes6[i], op[i], false); } + bool eRes_6[len0] = {true, true, false, false, false, true}; + for(int i = 0; i < len0; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_6[i], op[i], true); + } + for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes10[i], op[i], false); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i], false); + taosMemoryFree(rightData); + } + + + printf("--------------------json bool-- 0 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); + + key = "k8"; + bool eRes7[len+len1] = {false, false, true, true, false, false, false, true, false, false, false, false, false}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes7[i], op[i], false); + } + bool eRes_7[len0] = {true, true, false, false, false, false}; + for(int i = 0; i < len0; i++) { + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_7[i], op[i], true); + } + + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i], false); + taosMemoryFree(rightData); + } + + + printf("--------------------json string-- 6.6hello {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); + + key = "k9"; + bool eRes8[len+len1] = {true, false, false, false, false, true, false, true, true, false, true, true, true}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes8[i], op[i], false); + } + bool eRes_8[len0] = {false, true, true, true, false, true}; + for(int i = 0; i < len0; i++) { + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_8[i], op[i], true); + } + + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i], false); + taosMemoryFree(rightData); + } + + printf("---------------------json not exist-- NULL {1, 8, 2, 2, 3, 0, 0, 0, 0}------------------\n"); + + key = "k10"; // (NULL is true) return NULL, so use DBL_MAX represent NULL + double eRes9[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes9[i], op[i], false); + } + bool eRes_9[len0] = {false, false, false, false, false, false}; + for(int i = 0; i < len0; i++) { + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_9[i], op[i], true); + } + + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes9[i], op[i], false); taosMemoryFree(rightData); } From 02e46da0d1121616c32186764e3ef6e6f207fb1f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 5 Jun 2022 18:56:43 +0800 Subject: [PATCH 50/63] handle conflict --- source/dnode/vnode/src/meta/metaTable.c | 29 ++++++++++--------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index b6be4d1cf4..2d84ea3de9 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -61,31 +61,26 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const if (tTagToValArray((const STag *)data, &pTagVals) != 0) { return -1; } + char key[512] = {0}; + SIndexMultiTerm *terms = indexMultiTermCreate(); int16_t nCols = taosArrayGetSize(pTagVals); for (int i = 0; i < nCols; i++) { STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); char type = pTagVal->type; + sprintf(key, "%s_%s", tagName, pTagVal->pKey); + int32_t nKey = strlen(key); - char *Key = pTagVal->pKey; - char *key = taosMemoryCalloc(1, strlen(Key) + 2 + strlen(tagName)); - sprintf(key, "%s_%s", tagName, Key); - - int32_t nKey = strlen(key); SIndexTerm *term = NULL; - if (type == TSDB_DATA_TYPE_NULL) { + // handle null value } else if (type == TSDB_DATA_TYPE_NCHAR) { if (pTagVal->nData > 0) { - char * val = taosMemoryCalloc(1, pTagVal->nData); - int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val); - // printf("val: %s, len: %d", val, len); - - char *tval = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE); - memcpy(tval, (uint16_t *)&len, VARSTR_HEADER_SIZE); - memcpy(tval + VARSTR_HEADER_SIZE, val, len); + char * val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); + int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); + memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); type = TSDB_DATA_TYPE_VARCHAR; - term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, tval, len + 2); + term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, val, len); } else if (pTagVal->nData == 0) { char * val = NULL; int32_t len = 0; @@ -100,12 +95,10 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const int len = 0; term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len); } - if (term == NULL) { - // handle except later - } else { + if (term != NULL) { indexMultiTermAdd(terms, term); } - taosMemoryFree(key); + memset(key, 0, sizeof(key)); } tIndexJsonPut(pMeta->pTagIvtIdx, terms, tuid); indexMultiTermDestroy(terms); From e812d20a1bb24ec70bf285a93a0d19d77265988b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 5 Jun 2022 10:59:19 +0000 Subject: [PATCH 51/63] refact --- source/dnode/vnode/src/inc/tsdb.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ca80fe394a..171d52164c 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -352,9 +352,6 @@ typedef struct { TSDBKEY maxKey; } SBlock; -static FORCE_INLINE bool tsdbIsSupBlock(SBlock *pBlock) { return pBlock->numOfSubBlocks == 1; } -static FORCE_INLINE bool tsdbIsSubBlock(SBlock *pBlock) { return pBlock->numOfSubBlocks == 0; } - typedef struct { int32_t delimiter; // For recovery usage int32_t tid; From f02e10cf9941cfade47afcf762af8c82b20668a4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 5 Jun 2022 11:30:30 +0000 Subject: [PATCH 52/63] fix ci test --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 10b890262e..59a795b77a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -117,7 +117,6 @@ int32_t tsdbCommit(STsdb *pTsdb) { return -1; } -#if 0 // Skip expired memory data and expired FSET tsdbSeekCommitIter(&commith, commith.rtn.minKey); while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) { @@ -128,7 +127,6 @@ int32_t tsdbCommit(STsdb *pTsdb) { break; } } -#endif // commit fid = tsdbNextCommitFid(&(commith)); From 16f8450bbc30b688e52c79c13a45d2065f4caf45 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 5 Jun 2022 19:44:44 +0800 Subject: [PATCH 53/63] update test case --- source/libs/index/test/indexTests.cc | 113 ++++++++++++--------- source/libs/index/test/jsonUT.cc | 142 +++++++++++++++------------ 2 files changed, 142 insertions(+), 113 deletions(-) diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 74a30c3387..327a4ae219 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -498,10 +498,15 @@ TEST_F(IndexTFileEnv, test_tfile_write) { } taosArrayDestroy(data); - std::string colName("voltage"); - std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + std::string colName("voltage"); + std::string colVal("ab"); + + char buf[256] = {0}; + int16_t sz = colVal.size(); + memcpy(buf, (uint16_t*)&sz, 2); + memcpy(buf + 2, colVal.c_str(), colVal.size()); + SIndexTerm* term = + indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), buf, sizeof(buf)); SIndexTermQuery query = {term, QUERY_TERM}; SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); @@ -564,6 +569,18 @@ class IndexCacheEnv : public ::testing::Test { CacheObj* coj; }; +SIndexTerm* indexTermCreateT(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, + int32_t nColName, const char* colVal, int32_t nColVal) { + char buf[256] = {0}; + int16_t sz = nColVal; + memcpy(buf, (uint16_t*)&sz, 2); + memcpy(buf + 2, colVal, nColVal); + if (colType == TSDB_DATA_TYPE_BINARY) { + return indexTermCreate(suid, oper, colType, colName, nColName, buf, sizeof(buf)); + } else { + return indexTermCreate(suid, oper, colType, colName, nColName, colVal, nColVal); + } +} #define MAX_TERM_KEY_LEN 128 TEST_F(IndexCacheEnv, cache_test) { int version = 0; @@ -574,37 +591,37 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colName("voltage"); { std::string colVal("v1"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); // indexTermDestry(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v2"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } @@ -612,15 +629,15 @@ TEST_F(IndexCacheEnv, cache_test) { std::cout << "--------first----------" << std::endl; { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v4"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); indexTermDestroy(term); } @@ -630,8 +647,8 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colVal("v4"); for (size_t i = 0; i < 10; i++) { colVal[colVal.size() - 1] = 'a' + i; - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } @@ -640,8 +657,8 @@ TEST_F(IndexCacheEnv, cache_test) { // begin query { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; @@ -655,8 +672,8 @@ TEST_F(IndexCacheEnv, cache_test) { } { std::string colVal("v2"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; @@ -690,8 +707,8 @@ class IndexObj { return ret; } void Del(const std::string& colName, const std::string& colVal, uint64_t uid) { - SIndexTerm* term = indexTermCreate(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); Put(terms, uid); @@ -699,8 +716,8 @@ class IndexObj { } int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world", size_t numOfTable = 100 * 10000) { - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t i = 0; i < numOfTable; i++) { @@ -721,8 +738,8 @@ class IndexObj { // opt tColVal[taosRand() % colValSize] = 'a' + k % 26; } - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - tColVal.c_str(), tColVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + tColVal.c_str(), tColVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t j = 0; j < skip; j++) { @@ -757,8 +774,8 @@ class IndexObj { int SearchOne(const std::string& colName, const std::string& colVal) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); @@ -779,8 +796,8 @@ class IndexObj { } int SearchOneTarget(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); @@ -804,16 +821,16 @@ class IndexObj { void PutOne(const std::string& colName, const std::string& colVal) { SIndexMultiTerm* terms = indexMultiTermCreate(); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, 10); indexMultiTermDestroy(terms); } void PutOneTarge(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTerm* terms = indexMultiTermCreate(); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, val); indexMultiTermDestroy(terms); @@ -858,8 +875,8 @@ TEST_F(IndexEnv2, testIndexOpen) { { std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t i = 0; i < targetSize; i++) { @@ -873,8 +890,8 @@ TEST_F(IndexEnv2, testIndexOpen) { size_t size = 200; std::string colName("tag1"), colVal("hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t i = 0; i < size; i++) { @@ -888,8 +905,8 @@ TEST_F(IndexEnv2, testIndexOpen) { size_t size = 200; std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t i = size * 3; i < size * 4; i++) { @@ -903,8 +920,8 @@ TEST_F(IndexEnv2, testIndexOpen) { { std::string colName("tag1"), colVal("Hello"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); @@ -926,8 +943,8 @@ TEST_F(IndexEnv2, testEmptyIndexOpen) { { std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t i = 0; i < targetSize; i++) { diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 48ce8839c4..f795fbad01 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -19,6 +19,18 @@ static std::string dir = TD_TMP_DIR_PATH "json"; static std::string logDir = TD_TMP_DIR_PATH "log"; +SIndexTerm* indexTermCreateT(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, + int32_t nColName, const char* colVal, int32_t nColVal) { + char buf[256] = {0}; + int16_t sz = nColVal; + memcpy(buf, (uint16_t*)&sz, 2); + memcpy(buf + 2, colVal, nColVal); + if (colType == TSDB_DATA_TYPE_BINARY) { + return indexTermCreate(suid, oper, colType, colName, nColName, buf, sizeof(buf)); + } else { + return indexTermCreate(suid, oper, colType, colName, nColName, colVal, nColVal); + } +} static void initLog() { const char* defaultLogFileNamePrefix = "taoslog"; const int32_t maxLogFileNum = 10; @@ -59,8 +71,8 @@ class JsonEnv : public ::testing::Test { static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId, int8_t operType = ADD_VALUE) { - SIndexTerm* term = - indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen); + SIndexTerm* term = indexTermCreateT(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), + (const char*)data, dlen); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); tIndexJsonPut(index, terms, (int64_t)tableId); @@ -70,8 +82,8 @@ static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtyp static void delData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId, int8_t operType = DEL_VALUE) { - SIndexTerm* term = - indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen); + SIndexTerm* term = indexTermCreateT(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), + (const char*)data, dlen); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); tIndexJsonPut(index, terms, (int64_t)tableId); @@ -83,7 +95,7 @@ static void Search(SIndexJson* index, const std::string& colNam, int8_t dtype, v std::string colName(colNam); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, dtype, colName.c_str(), colName.size(), (const char*)data, dlen); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, dtype, colName.c_str(), colName.size(), (const char*)data, dlen); SArray* res = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, (EIndexQueryType)filterType); @@ -95,8 +107,8 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("test"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -108,8 +120,8 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("voltage"); std::string colVal("ab1"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -121,8 +133,8 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("voltage"); std::string colVal("123"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -136,8 +148,8 @@ TEST_F(JsonEnv, testWrite) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_TERM); @@ -150,8 +162,8 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("test"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -165,8 +177,8 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("abxxxxxxxxxxxx"); for (int i = 0; i < 10; i++) { colVal[i % colVal.size()] = '0' + i % 128; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -179,8 +191,8 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("voltagefdadfa"); std::string colVal("abxxxxxxxxxxxx"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -194,8 +206,8 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_TERM); @@ -209,8 +221,8 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN); @@ -224,8 +236,8 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL); @@ -241,8 +253,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { std::string colName("test"); // std::string colVal("10"); int val = 10; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -254,8 +266,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { { std::string colName("test2"); int val = 20; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -267,8 +279,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { { std::string colName("test"); int val = 15; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -280,8 +292,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { { std::string colName("test2"); const char* val = "test"; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - (const char*)val, strlen(val)); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + (const char*)val, strlen(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -294,8 +306,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { std::string colName("test"); int val = 15; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_TERM); @@ -308,8 +320,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { int val = 15; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN); @@ -323,8 +335,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { ; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(int)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(int)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL); @@ -338,8 +350,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { // std::string colVal("10"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN); @@ -353,8 +365,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { // std::string colVal("10"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL); @@ -368,8 +380,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { { std::string colName("test1"); int val = 10; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -381,8 +393,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { { std::string colName("test"); std::string colVal("xxxxxxxxxxxxxxxxxxx"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -396,8 +408,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { int val = 10; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_TERM); @@ -410,8 +422,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { int val = 10; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(int)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(int)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN); @@ -425,8 +437,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { int val = 10; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL); @@ -439,8 +451,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { int val = 10; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN); @@ -453,8 +465,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { int val = 10; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL); @@ -466,8 +478,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { std::string colName("other_column"); int val = 100; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -482,8 +494,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { // std::string colVal("10"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN); @@ -494,8 +506,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { { std::string colName("test1"); int val = 15; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -510,8 +522,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { // std::string colVal("10"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL); From 1c46958cb4813dad693b066d42838c1262c505b8 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 5 Jun 2022 19:55:06 +0800 Subject: [PATCH 54/63] enh: check max/sum/avg func for rsma --- include/common/tdatablock.h | 2 +- source/common/src/tdatablock.c | 38 +++++++++++++--------- source/dnode/vnode/src/sma/smaRollup.c | 8 +++-- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 4 +-- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +++- 5 files changed, 36 insertions(+), 22 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index cce763cafe..bf16dd2fc4 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -232,7 +232,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); -void blockDebugShowData(const SArray* dataBlocks); +void blockDebugShowData(const SArray* dataBlocks, const char* flag); int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, tb_uid_t suid); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 055992db53..909fee09aa 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1488,7 +1488,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { return buf; } -void blockDebugShowData(const SArray* dataBlocks) { +void blockDebugShowData(const SArray* dataBlocks, const char* flag) { char pBuf[128] = {0}; int32_t sz = taosArrayGetSize(dataBlocks); for (int32_t i = 0; i < sz; i++) { @@ -1496,7 +1496,7 @@ void blockDebugShowData(const SArray* dataBlocks) { int32_t colNum = pDataBlock->info.numOfCols; int32_t rows = pDataBlock->info.rows; for (int32_t j = 0; j < rows; j++) { - printf("|"); + printf("%s |", flag); for (int32_t k = 0; k < colNum; k++) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); @@ -1521,8 +1521,11 @@ void blockDebugShowData(const SArray* dataBlocks) { case TSDB_DATA_TYPE_UBIGINT: printf(" %15lu |", *(uint64_t*)var); break; + case TSDB_DATA_TYPE_FLOAT: + printf(" %15f |", *(float*)var); + break; case TSDB_DATA_TYPE_DOUBLE: - printf(" %15f |", *(double*)var); + printf(" %15lf |", *(double*)var); break; } } @@ -1550,8 +1553,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks bufSize += sizeof(SSubmitBlk); } - ASSERT(bufSize < 3 * 1024 * 1024); - *pReq = taosMemoryCalloc(1, bufSize); if (!(*pReq)) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1562,7 +1563,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks int32_t msgLen = sizeof(SSubmitReq); int32_t numOfBlks = 0; SRowBuilder rb = {0}; - tdSRowInit(&rb, pTSchema->version); // TODO: use the latest version + tdSRowInit(&rb, pTSchema->version); for (int32_t i = 0; i < sz; ++i) { SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); @@ -1580,18 +1581,17 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks pSubmitBlk->uid = pDataBlock->info.groupId; pSubmitBlk->numOfRows = rows; - ++numOfBlks; - msgLen += sizeof(SSubmitBlk); int32_t dataLen = 0; for (int32_t j = 0; j < rows; ++j) { // iterate by row tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf - printf("|"); bool isStartKey = false; int32_t offset = 0; for (int32_t k = 0; k < colNum; ++k) { // iterate by column SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + STColumn* pCol = &pTSchema->columns[k]; + ASSERT(pCol->type == pColInfoData->info.type); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: if (!isStartKey) { @@ -1600,29 +1600,29 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks offset, k); } else { - tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, offset, k); + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, offset, k); } break; case TSDB_DATA_TYPE_NCHAR: { - tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true, offset, k); + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true, offset, k); break; } case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY - tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true, offset, k); + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true, offset, k); break; } case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: case TSDB_DATA_TYPE_MEDIUMBLOB: - printf("the column type %" PRIi16 " is defined but not implemented yet\n", pColInfoData->info.type); + uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); TASSERT(0); break; default: if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { - tdAppendColValToRow(&rb, 2, pColInfoData->info.type, TD_VTYPE_NORM, var, true, offset, k); + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pColInfoData->info.type, TD_VTYPE_NORM, var, true, offset, k); } else { - printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); TASSERT(0); } break; @@ -1630,7 +1630,13 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks offset += TYPE_BYTES[pColInfoData->info.type]; } dataLen += TD_ROW_LEN(rb.pBuf); +#ifdef TD_DEBUG_PRINT_ROW + tdSRowPrint(rb.pBuf, pTSchema, __func__); +#endif } + + ++numOfBlks; + pSubmitBlk->dataLen = dataLen; msgLen += pSubmitBlk->dataLen; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 80c8d20572..e738d3a408 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -400,7 +400,11 @@ static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int3 } if (taosArrayGetSize(pResult) > 0) { - blockDebugShowData(pResult); +#if 1 + char flag[10] = {0}; + snprintf(flag, 10, "level %" PRIi8, level); + blockDebugShowData(pResult, flag); +#endif STsdb *sinkTsdb = (level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2); SSubmitReq *pReq = NULL; if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, SMA_VID(pSma), suid) != 0) { @@ -444,7 +448,7 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb } if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { - // TODO: use the proper schema instead of 0, and cache STSchema in cache + // TODO: cache STSchema STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, -1); if (!pTSchema) { terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 350e723541..fa392baa16 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -297,8 +297,8 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo tSkipListPutBatchByIter(pTbData->pData, &blkIter, (iter_next_fn_t)tGetSubmitBlkNext); #ifdef TD_DEBUG_PRINT_ROW - printf("!!! %s:%d table %" PRIi64 " has %d rows in skiplist\n\n", __func__, __LINE__, pTbData->uid, - SL_SIZE(pTbData->pData)); + printf("!!! %s:%d vgId:%d dir:%s table:%" PRIi64 " has %d rows in skiplist\n\n", __func__, __LINE__, + TD_VID(pTsdb->pVnode), pTsdb->dir, pTbData->uid, SL_SIZE(pTbData->pData)); #endif // Set statistics diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5f2ea80078..b50dc95ca1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1661,7 +1661,11 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa } #ifdef TD_DEBUG_PRINT_ROW - tdSRowPrint(row1, pSchema1, __func__); + char flags[70] = {0}; + STsdb* pTsdb = pTsdbReadHandle->rhelper.pRepo; + snprintf(flags, 70, "%s:%d vgId:%d dir:%s row1%s=NULL,row2%s=NULL", __func__, __LINE__, TD_VID(pTsdb->pVnode), + pTsdb->dir, row1 ? "!" : "", row2 ? "!" : ""); + tdSRowPrint(row1, pSchema1, flags); #endif if (isRow1DataRow) { From d01bb6a8236562a75020f29fca02b05cbc3e5925 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sun, 5 Jun 2022 20:44:13 +0800 Subject: [PATCH 55/63] fix(os): add other two win32 udf case --- tests/pytest/util/dnodes.py | 2 +- tests/system-test/0-others/udf_create.py | 62 +++++++++-------- .../system-test/0-others/udf_restart_taosd.py | 66 +++++++++++-------- 3 files changed, 73 insertions(+), 57 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 07518219ce..b6ffda5c86 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -318,7 +318,7 @@ class TDDnode: print(cmd) if (not self.remoteIP == ""): - self.remoteExec(self.cfgDict, "tdDnodes.deploy(%d,updateCfgDict)\ntdDnodes.start(%d)"%(self.index, self.index)) + self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].deployed=1\ntdDnodes.dnodes[%d].logDir=\"%%s/sim/dnode%%d/log\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.dnodes[%d].cfgDir=\"%%s/sim/dnode%%d/cfg\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.start(%d)"%(self.index-1,self.index-1,self.index-1,self.index,self.index-1,self.index-1,self.index,self.index)) self.running = 1 else: if os.system(cmd) != 0: diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index e2c6e3c10b..88ed667161 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -25,7 +25,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] @@ -41,11 +41,19 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] print(projPath) - libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") - libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") - os.system("mkdir /tmp/udf/") - os.system("cp %s /tmp/udf/ "%libudf1.replace("\n" ,"")) - os.system("cp %s /tmp/udf/ "%libudf2.replace("\n" ,"")) + if platform.system().lower() == 'windows': + self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + if (not tdDnodes.dnodes[0].remoteIP == ""): + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\") + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\") + self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so') + self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so') + else: + self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf1 = self.libudf1.replace('\r','').replace('\n','') + self.libudf2 = self.libudf2.replace('\r','').replace('\n','') def prepare_data(self): @@ -136,11 +144,11 @@ class TDTestCase: for i in range(5): # create scalar functions - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;") + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) # create aggregate functions - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;") + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) functions = tdSql.getResult("show functions") function_nums = len(functions) @@ -161,11 +169,11 @@ class TDTestCase: tdLog.info("drop two udf functions success ") # create scalar functions - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;") + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) # create aggregate functions - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;") + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) functions = tdSql.getResult("show functions") function_nums = len(functions) @@ -533,8 +541,8 @@ class TDTestCase: tdSql.query("drop function udf2 ") # create function without buffer - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int") - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double") + tdSql.execute("create function udf1 as '%s' outputtype int"%self.libudf1) + tdSql.execute("create aggregate function udf2 as '%s' outputtype double"%self.libudf2) udf1_sqls ,udf2_sqls = self.try_query_sql() for scalar_sql in udf1_sqls: @@ -549,8 +557,8 @@ class TDTestCase: tdSql.query("drop function udf2 ") # create function without buffer - tdSql.execute("create aggregate function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") - tdSql.execute("create function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8") + tdSql.execute("create aggregate function udf1 as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute("create function udf2 as '%s' outputtype double bufSize 8"%self.libudf2) udf1_sqls ,udf2_sqls = self.try_query_sql() for scalar_sql in udf1_sqls: @@ -558,8 +566,8 @@ class TDTestCase: for aggregate_sql in udf2_sqls: tdSql.error(aggregate_sql) - tdSql.execute(" create function db as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") - tdSql.execute(" create aggregate function test as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") + tdSql.execute(" create function db as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute(" create aggregate function test as '%s' outputtype int bufSize 8 "%self.libudf1) tdSql.error(" select db(c1) from stb1 ") tdSql.error(" select db(c1,c6), db(c6) from stb1 ") tdSql.error(" select db(num1,num2), db(num1) from tb ") @@ -607,17 +615,17 @@ class TDTestCase: tdLog.info(" create function name is not build_in functions ") tdSql.execute(" drop function udf1 ") tdSql.execute(" drop function udf2 ") - tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8") - tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8") - tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function tbname as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function function as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function stable as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function union as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function 123 as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function 123db as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function mnode as '/tmp/udf/libudf2.so' outputtype double bufSize 8") + tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function tbname as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function function as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function stable as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function union as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function 123 as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function 123db as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function mnode as '%s' outputtype double bufSize 8"%self.libudf2) def restart_taosd_query_udf(self): diff --git a/tests/system-test/0-others/udf_restart_taosd.py b/tests/system-test/0-others/udf_restart_taosd.py index 24d3b5a9c3..a8aa447026 100644 --- a/tests/system-test/0-others/udf_restart_taosd.py +++ b/tests/system-test/0-others/udf_restart_taosd.py @@ -25,7 +25,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] @@ -41,11 +41,19 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] print(projPath) - libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") - libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") - os.system("mkdir /tmp/udf/") - os.system("cp %s /tmp/udf/ "%libudf1.replace("\n" ,"")) - os.system("cp %s /tmp/udf/ "%libudf2.replace("\n" ,"")) + if platform.system().lower() == 'windows': + self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + if (not tdDnodes.dnodes[0].remoteIP == ""): + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\") + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\") + self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so') + self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so') + else: + self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf1 = self.libudf1.replace('\r','').replace('\n','') + self.libudf2 = self.libudf2.replace('\r','').replace('\n','') def prepare_data(self): @@ -136,11 +144,11 @@ class TDTestCase: for i in range(5): # create scalar functions - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;") + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) # create aggregate functions - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;") + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) functions = tdSql.getResult("show functions") function_nums = len(functions) @@ -161,11 +169,11 @@ class TDTestCase: tdLog.info("drop two udf functions success ") # create scalar functions - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;") + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) # create aggregate functions - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;") + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) functions = tdSql.getResult("show functions") function_nums = len(functions) @@ -533,8 +541,8 @@ class TDTestCase: tdSql.query("drop function udf2 ") # create function without buffer - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int") - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double") + tdSql.execute("create function udf1 as '%s' outputtype int"%self.libudf1) + tdSql.execute("create aggregate function udf2 as '%s' outputtype double"%self.libudf2) udf1_sqls ,udf2_sqls = self.try_query_sql() for scalar_sql in udf1_sqls: @@ -549,8 +557,8 @@ class TDTestCase: tdSql.query("drop function udf2 ") # create function without buffer - tdSql.execute("create aggregate function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") - tdSql.execute("create function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8") + tdSql.execute("create aggregate function udf1 as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute("create function udf2 as '%s' outputtype double bufSize 8"%self.libudf2) udf1_sqls ,udf2_sqls = self.try_query_sql() for scalar_sql in udf1_sqls: @@ -558,8 +566,8 @@ class TDTestCase: for aggregate_sql in udf2_sqls: tdSql.error(aggregate_sql) - tdSql.execute(" create function db as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") - tdSql.execute(" create aggregate function test as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") + tdSql.execute(" create function db as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute(" create aggregate function test as '%s' outputtype int bufSize 8 "%self.libudf1) tdSql.error(" select db(c1) from stb1 ") tdSql.error(" select db(c1,c6), db(c6) from stb1 ") tdSql.error(" select db(num1,num2), db(num1) from tb ") @@ -607,17 +615,17 @@ class TDTestCase: tdLog.info(" create function name is not build_in functions ") tdSql.execute(" drop function udf1 ") tdSql.execute(" drop function udf2 ") - tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8") - tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8") - tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function tbname as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function function as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function stable as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function union as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function 123 as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function 123db as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function mnode as '/tmp/udf/libudf2.so' outputtype double bufSize 8") + tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function tbname as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function function as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function stable as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function union as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function 123 as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function 123db as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function mnode as '%s' outputtype double bufSize 8"%self.libudf2) def restart_taosd_query_udf(self): @@ -629,8 +637,8 @@ class TDTestCase: tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,169.661427555) tdSql.checkData(0,1,169.661427555) - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) time.sleep(2) From 85e0eee1b2fc0e740f2ff1e4fad31b50c11f0e3f Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sun, 5 Jun 2022 20:49:34 +0800 Subject: [PATCH 56/63] fix(os): add other two win32 udf case --- tests/system-test/0-others/udf_restart_taosd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/0-others/udf_restart_taosd.py b/tests/system-test/0-others/udf_restart_taosd.py index a8aa447026..94b14f01ba 100644 --- a/tests/system-test/0-others/udf_restart_taosd.py +++ b/tests/system-test/0-others/udf_restart_taosd.py @@ -637,8 +637,8 @@ class TDTestCase: tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,169.661427555) tdSql.checkData(0,1,169.661427555) - # tdDnodes.stop(1) - # tdDnodes.start(1) + tdDnodes.stop(1) + tdDnodes.start(1) time.sleep(2) From b1a332b5c24f0000d2a299ca3872b7ad8cf1fbe8 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sun, 5 Jun 2022 21:16:56 +0800 Subject: [PATCH 57/63] fix(os): add other two win32 udf case --- tests/system-test/fulltest.bat | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.bat b/tests/system-test/fulltest.bat index f4da0aabb3..ea57bc7427 100644 --- a/tests/system-test/fulltest.bat +++ b/tests/system-test/fulltest.bat @@ -2,4 +2,6 @@ python3 .\test.py -f 0-others\taosShell.py python3 .\test.py -f 0-others\taosShellError.py python3 .\test.py -f 0-others\taosShellNetChk.py -python3 .\test.py -f 0-others\udfTest.py \ No newline at end of file +python3 .\test.py -f 0-others\udfTest.py +python3 .\test.py -f 0-others\udf_create.py +python3 .\test.py -f 0-others\udf_restart_taosd.py \ No newline at end of file From 30b8bd1b16c8f6aaba3ade41a781e378ce07c4be Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 5 Jun 2022 21:31:54 +0800 Subject: [PATCH 58/63] feat: add sort/group logic for json --- source/libs/scalar/src/filter.c | 3 +- source/libs/scalar/src/sclvector.c | 48 +++++++++++++------ .../libs/scalar/test/scalar/scalarTests.cpp | 10 ++-- 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 195ec8a577..2a23cbeed5 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -30,7 +30,8 @@ OptrStr gOptrStr[] = { {OP_TYPE_MULTI, "*"}, {OP_TYPE_DIV, "/"}, {OP_TYPE_MOD, "%"}, - + {OP_TYPE_MINUS, "minus"}, + {OP_TYPE_ASSIGN, "assign"}, // bit operator {OP_TYPE_BIT_AND, "&"}, {OP_TYPE_BIT_OR, "|"}, diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 85778e6a35..d63a9ebb48 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -452,13 +452,13 @@ double getVectorDoubleValue_JSON(void *src, int32_t index){ return out; } -void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull){ +bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull){ if(optr == OP_TYPE_JSON_CONTAINS) { - return; + return true; } if(typeLeft != TSDB_DATA_TYPE_JSON && typeRight != TSDB_DATA_TYPE_JSON){ - return; + return true; } if(typeLeft == TSDB_DATA_TYPE_JSON){ @@ -469,15 +469,22 @@ void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t typeRight = **pRightData; (*pRightData) ++; } + + if(optr == OP_TYPE_LIKE || optr == OP_TYPE_NOT_LIKE || optr == OP_TYPE_MATCH || optr == OP_TYPE_NMATCH){ + if(typeLeft != TSDB_DATA_TYPE_NCHAR && typeLeft != TSDB_DATA_TYPE_BINARY){ + return false; + } + } + if(typeLeft == TSDB_DATA_TYPE_NULL || typeRight == TSDB_DATA_TYPE_NULL){ *isNull = true; - return; + return true; } int8_t type = vectorGetConvertType(typeLeft, typeRight); if(type == 0) { *fp = filterGetCompFunc(typeLeft, optr); - return; + return true; } *fp = filterGetCompFunc(type, optr); @@ -497,6 +504,7 @@ void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t convertNumberToNumber(*pRightData, pRightOut, typeRight, type); *pRightData = pRightOut; } + return true; } int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int16_t inType, int16_t outType) { @@ -1533,12 +1541,16 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * int64_t leftOut = 0; int64_t rightOut = 0; bool isJsonnull = false; - convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); + bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); if(isJsonnull){ ASSERT(0); } - bool res = filterDoCompare(fp, optr, pLeftData, pRightData); - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); + if(!result){ + colDataAppendInt8(pOut->columnData, i, (int8_t*)&result); + }else{ + bool res = filterDoCompare(fp, optr, pLeftData, pRightData); + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); + } } } else if (pRight->numOfRows == 1) { ASSERT(pLeft->pHashFilter == NULL); @@ -1554,12 +1566,16 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * int64_t leftOut = 0; int64_t rightOut = 0; bool isJsonnull = false; - convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); + bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); if(isJsonnull){ ASSERT(0); } - bool res = filterDoCompare(fp, optr, pLeftData, pRightData); - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); + if(!result){ + colDataAppendInt8(pOut->columnData, i, (int8_t*)&result); + }else{ + bool res = filterDoCompare(fp, optr, pLeftData, pRightData); + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); + } } } else if (pLeft->numOfRows == 1) { for (; i >= 0 && i < pRight->numOfRows; i += step) { @@ -1574,12 +1590,16 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * int64_t leftOut = 0; int64_t rightOut = 0; bool isJsonnull = false; - convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); + bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); if(isJsonnull){ ASSERT(0); } - bool res = filterDoCompare(fp, optr, pLeftData, pRightData); - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); + if(!result){ + colDataAppendInt8(pOut->columnData, i, (int8_t*)&result); + }else{ + bool res = filterDoCompare(fp, optr, pLeftData, pRightData); + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); + } } } } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 502ec17ead..2912fce8be 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -1382,7 +1382,7 @@ TEST(columnTest, json_column_logic_op) { printf("--------------------json int-- -10 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); key = "k6"; - bool eRes5[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, true}; + bool eRes5[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i], false); } @@ -1411,7 +1411,7 @@ TEST(columnTest, json_column_logic_op) { for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i], false); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes6[i], op[i], false); taosMemoryFree(rightData); } @@ -1430,7 +1430,7 @@ TEST(columnTest, json_column_logic_op) { for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i], false); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes7[i], op[i], false); taosMemoryFree(rightData); } @@ -1438,7 +1438,7 @@ TEST(columnTest, json_column_logic_op) { printf("--------------------json string-- 6.6hello {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); key = "k9"; - bool eRes8[len+len1] = {true, false, false, false, false, true, false, true, true, false, true, true, true}; + bool eRes8[len+len1] = {true, false, false, false, false, true, false, true, true, false, true, false, true}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes8[i], op[i], false); } @@ -1449,7 +1449,7 @@ TEST(columnTest, json_column_logic_op) { for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i], false); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes8[i], op[i], false); taosMemoryFree(rightData); } From 0ca6a5ae271a4703da44f99fd2e8c067c6374238 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 5 Jun 2022 21:46:07 +0800 Subject: [PATCH 59/63] refactor json index --- cmake/cmake.options | 2 +- include/libs/index/index.h | 3 +- source/dnode/vnode/inc/vnode.h | 19 ++++----- source/dnode/vnode/src/inc/meta.h | 11 +++--- source/dnode/vnode/src/inc/vnodeInt.h | 3 +- source/dnode/vnode/src/meta/metaOpen.c | 6 --- source/dnode/vnode/src/meta/metaQuery.c | 4 -- source/dnode/vnode/src/meta/metaTable.c | 24 +----------- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +++ source/libs/executor/src/executorimpl.c | 19 ++++----- source/libs/index/src/indexFilter.c | 52 ++++++++++--------------- 11 files changed, 59 insertions(+), 90 deletions(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index ab3c5ac1ad..62816a80d5 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -146,6 +146,6 @@ option( option( BUILD_WITH_INVERTEDINDEX "If use invertedIndex" - OFF + ON ) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 180c7e7216..bd601f1d9f 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -193,8 +193,9 @@ void indexInit(); /* index filter */ typedef struct SIndexMetaArg { - void* metaHandle; void* metaEx; + void* idx; + void* ivtIdx; uint64_t suid; } SIndexMetaArg; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 7bd2eadd4b..8c7a78a5af 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -56,7 +56,7 @@ int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg); int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp); int32_t vnodeProcessCMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); -int32_t vnodePreprocessQueryMsg(SVnode * pVnode, SRpcMsg * pMsg); +int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); @@ -86,7 +86,7 @@ typedef struct SMetaFltParam { tb_uid_t suid; int16_t cid; int16_t type; - char *val; + char * val; bool reverse; int (*filterFunc)(void *a, void *b, int16_t type); @@ -121,7 +121,8 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo bool isTsdbCacheLastRow(tsdbReaderT *pReader); int32_t tsdbGetAllTableList(SMeta *pMeta, uint64_t uid, SArray *list); int32_t tsdbGetCtbIdList(SMeta *pMeta, int64_t suid, SArray *list); -void *tsdbGetIdx(SMeta *pMeta); +void * tsdbGetIdx(SMeta *pMeta); +void * tsdbGetIvtIdx(SMeta *pMeta); int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle); bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle); @@ -197,7 +198,7 @@ struct SMetaEntry { int64_t version; int8_t type; tb_uid_t uid; - char *name; + char * name; union { struct { SSchemaWrapper schemaRow; @@ -225,17 +226,17 @@ struct SMetaEntry { struct SMetaReader { int32_t flags; - SMeta *pMeta; + SMeta * pMeta; SDecoder coder; SMetaEntry me; - void *pBuf; + void * pBuf; int32_t szBuf; }; struct SMTbCursor { - TBC *pDbc; - void *pKey; - void *pVal; + TBC * pDbc; + void * pKey; + void * pVal; int32_t kLen; int32_t vLen; SMetaReader mr; diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index b610676c19..63a8e7ec68 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -69,12 +69,11 @@ struct SMeta { TTB* pUidIdx; TTB* pNameIdx; TTB* pCtbIdx; -#ifdef USE_INVERTED_INDEX + // ivt idx and idx void* pTagIvtIdx; -#else - TTB* pTagIdx; -#endif - TTB* pTtlIdx; + TTB* pTagIdx; + TTB* pTtlIdx; + TTB* pSmaIdx; SMetaIdx* pIdx; }; @@ -117,7 +116,7 @@ typedef struct { } SSmaIdxKey; // metaTable ================== -int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, +int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, STagIdxKey** ppTagIdxKey, int32_t* nTagIdxKey); #ifndef META_REFACT diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index d3b5f29aac..b35f6e3abe 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -87,7 +87,7 @@ int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* p int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq); int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids); -int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp *pMetaRsp); +int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp); SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); int metaGetTableEntryByName(SMetaReader* pReader, const char* name); @@ -104,6 +104,7 @@ int32_t metaSnapshotReaderOpen(SMeta* pMeta, SMetaSnapshotReader** ppRea int32_t metaSnapshotReaderClose(SMetaSnapshotReader* pReader); int32_t metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t* nData); void* metaGetIdx(SMeta* pMeta); +void* metaGetIvtIdx(SMeta* pMeta); int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 86637d2850..95510c25e5 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -93,7 +93,6 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { } // open pTagIdx -#ifdef USE_INVERTED_INDEX // TODO(yihaoDeng), refactor later char indexFullPath[128] = {0}; sprintf(indexFullPath, "%s/%s", pMeta->path, "invert"); @@ -104,13 +103,11 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { goto _err; } -#else ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx); if (ret < 0) { metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } -#endif // open pTtlIdx ret = tdbTbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx); @@ -141,11 +138,8 @@ _err: if (pMeta->pIdx) metaCloseIdx(pMeta); if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx); -#ifdef USE_INVERTED_INDEX if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx); -#else if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); -#endif if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index fc7d1c3c92..21a55d6463 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -603,9 +603,6 @@ typedef struct { } SIdxCursor; int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { -#ifdef USE_INVERTED_INDEX - return -1; -#else SIdxCursor *pCursor = NULL; int32_t ret = 0, valid = 0; @@ -681,5 +678,4 @@ END: taosMemoryFree(pCursor); return ret; -#endif } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 2d84ea3de9..4f4a940bc7 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -890,27 +890,12 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; return metaSaveJsonVarToIdx(pMeta, pCtbEntry, pTagColumn); } - - // update tag index -#ifdef USE_INVERTED_INDEX - tb_uid_t suid = pCtbEntry->ctbEntry.suid; - tb_uid_t tuid = pCtbEntry->uid; - - SIndexMultiTerm *tmGroup = indexMultiTermCreate(); - - SIndexTerm *tm = indexTermCreate(suid, ADD_VALUE, pTagColumn->type, pTagColumn->name, sizeof(pTagColumn->name), - pTagData, pTagData == NULL ? 0 : strlen(pTagData)); - indexMultiTermAdd(tmGroup, tm); - int ret = indexPut((SIndex *)pMeta->pTagIvtIdx, tmGroup, tuid); - indexMultiTermDestroy(tmGroup); -#else if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { return -1; } tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); metaDestroyTagIdxKey(pTagIdxKey); -#endif tDecoderClear(&dc); tdbFree(pData); return 0; @@ -995,10 +980,5 @@ _err: return -1; } // refactor later -void *metaGetIdx(SMeta *pMeta) { -#ifdef USE_INVERTED_INDEX - return pMeta->pTagIvtIdx; -#else - return pMeta->pTagIdx; -#endif -} +void *metaGetIdx(SMeta *pMeta) { return pMeta->pTagIdx; } +void *metaGetIvtIdx(SMeta *pMeta) { return pMeta->pTagIvtIdx; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5f2ea80078..b2c077e9f5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2829,6 +2829,12 @@ void* tsdbGetIdx(SMeta* pMeta) { } return metaGetIdx(pMeta); } +void* tsdbGetIvtIdx(SMeta* pMeta) { + if (pMeta == NULL) { + return NULL; + } + return metaGetIvtIdx(pMeta); +} int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) { SMCtbCursor* pCur = metaOpenCtbCursor(pMeta, uid); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 07f4251340..bee0ce7f89 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4052,12 +4052,12 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p goto _error; } - pInfo->limit = *pLimit; - pInfo->slimit = *pSlimit; - pInfo->curOffset = pLimit->offset; + pInfo->limit = *pLimit; + pInfo->slimit = *pSlimit; + pInfo->curOffset = pLimit->offset; pInfo->curSOffset = pSlimit->offset; pInfo->binfo.pRes = pResBlock; - pInfo->pFilterNode= pCondition; + pInfo->pFilterNode = pCondition; int32_t numOfCols = num; int32_t numOfRows = 4096; @@ -4404,7 +4404,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; STimeWindowAggSupp twSup = { - .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN}; + .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN}; tsdbReaderT pDataReader = NULL; if (pHandle->vnode) { pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond); @@ -4487,7 +4487,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); SLimit limit = {.limit = pProjPhyNode->limit, .offset = pProjPhyNode->offset}; SLimit slimit = {.limit = pProjPhyNode->slimit, .offset = pProjPhyNode->soffset}; - pOptr = createProjectOperatorInfo(ops[0], pExprInfo, num, pResBlock, &limit, &slimit, pProjPhyNode->node.pConditions, pTaskInfo); + pOptr = createProjectOperatorInfo(ops[0], pExprInfo, num, pResBlock, &limit, &slimit, + pProjPhyNode->node.pConditions, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_AGG == type) { SAggPhysiNode* pAggNode = (SAggPhysiNode*)pPhyNode; SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); @@ -4836,7 +4837,8 @@ int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STa if (tableType == TSDB_SUPER_TABLE) { if (pTagCond) { - SIndexMetaArg metaArg = {.metaEx = metaHandle, .metaHandle = tsdbGetIdx(metaHandle), .suid = tableUid}; + SIndexMetaArg metaArg = { + .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid}; SArray* res = taosArrayInit(8, sizeof(uint64_t)); code = doFilterTag(pTagCond, &metaArg, res); @@ -5187,8 +5189,7 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo return TSDB_CODE_SUCCESS; } -int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, - size_t size) { +int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, size_t size) { pSup->keySize = sizeof(int64_t) + sizeof(TSKEY); pSup->pKeyBuf = taosMemoryCalloc(1, pSup->keySize); pSup->pResultRows = taosArrayInit(1024, size); diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index cb605ca073..34251ab918 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -335,44 +335,34 @@ static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) { } static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { SIndexMetaArg *arg = &output->arg; -#ifdef USE_INVERTED_INDEX - SIndexTerm *tm = indexTermCreate(arg->suid, DEFAULT, right->colValType, left->colName, strlen(left->colName), - right->condValue, strlen(right->condValue)); - if (tm == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } + int ret = 0; - int ret = 0; EIndexQueryType qtype = 0; SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); - - SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); - indexMultiTermQueryAdd(mtm, tm, qtype); if (left->colValType == TSDB_DATA_TYPE_JSON) { - ret = tIndexJsonSearch(arg->metaHandle, mtm, output->result); + SIndexTerm *tm = indexTermCreate(arg->suid, DEFAULT, right->colValType, left->colName, strlen(left->colName), + right->condValue, strlen(right->condValue)); + if (tm == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); + indexMultiTermQueryAdd(mtm, tm, qtype); + ret = tIndexJsonSearch(arg->ivtIdx, mtm, output->result); } else { - ret = indexSearch(arg->metaHandle, mtm, output->result); + bool reverse; + Filter filterFunc = sifGetFilterFunc(qtype, &reverse); + + SMetaFltParam param = {.suid = arg->suid, + .cid = left->colId, + .type = left->colValType, + .val = right->condValue, + .reverse = reverse, + .filterFunc = filterFunc}; + + ret = metaFilteTableIds(arg->metaEx, ¶m, output->result); } - indexDebug("index filter data size: %d", (int)taosArrayGetSize(output->result)); - indexMultiTermQueryDestroy(mtm); return ret; -#else - EIndexQueryType qtype = 0; - SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); - bool reverse; - Filter filterFunc = sifGetFilterFunc(qtype, &reverse); - - SMetaFltParam param = {.suid = arg->suid, - .cid = left->colId, - .type = left->colValType, - .val = right->condValue, - .reverse = reverse, - .filterFunc = filterFunc}; - - int ret = metaFilteTableIds(arg->metaEx, ¶m, output->result); - return ret; -#endif - return 0; } static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { From a3a959043ea09943b7ada3aeab24983ad696ba26 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 5 Jun 2022 22:25:10 +0800 Subject: [PATCH 60/63] feat: test arith/logic operator for json --- include/util/tdef.h | 2 +- source/libs/nodes/src/nodesUtilFuncs.c | 2 +- source/libs/parser/inc/sql.y | 2 +- source/libs/parser/src/sql.c | 2 +- source/libs/scalar/src/filter.c | 2 +- source/libs/scalar/src/sclvector.c | 35 +++++++++++-------- .../libs/scalar/test/scalar/scalarTests.cpp | 6 ++-- 7 files changed, 28 insertions(+), 23 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 0ae22d1953..ebfb7b696a 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -129,7 +129,7 @@ typedef enum EOperatorType { OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV, - OP_TYPE_MOD, + OP_TYPE_REM, // unary arithmetic operator OP_TYPE_MINUS, OP_TYPE_ASSIGN, diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 76f15afc8e..d160cb8f44 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1089,7 +1089,7 @@ bool nodesIsArithmeticOp(const SOperatorNode* pOp) { case OP_TYPE_SUB: case OP_TYPE_MULTI: case OP_TYPE_DIV: - case OP_TYPE_MOD: + case OP_TYPE_REM: return true; default: break; diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 6c090a0790..5e8ab9b9aa 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -608,7 +608,7 @@ expression(A) ::= expression(B) NK_SLASH expression(C). expression(A) ::= expression(B) NK_REM expression(C). { SToken s = getTokenFromRawExprNode(pCxt, B); SToken e = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); } expression(A) ::= column_reference(B) NK_ARROW NK_STRING(C). { SToken s = getTokenFromRawExprNode(pCxt, B); diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index ff4fe4032e..cd84495b9b 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -4540,7 +4540,7 @@ static YYACTIONTYPE yy_reduce( { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy686); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy686); - yylhsminor.yy686 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), releaseRawExprNode(pCxt, yymsp[0].minor.yy686))); + yylhsminor.yy686 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), releaseRawExprNode(pCxt, yymsp[0].minor.yy686))); } yymsp[-2].minor.yy686 = yylhsminor.yy686; break; diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 2a23cbeed5..5fdbbfe810 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -29,7 +29,7 @@ OptrStr gOptrStr[] = { {OP_TYPE_SUB, "-"}, {OP_TYPE_MULTI, "*"}, {OP_TYPE_DIV, "/"}, - {OP_TYPE_MOD, "%"}, + {OP_TYPE_REM, "%"}, {OP_TYPE_MINUS, "minus"}, {OP_TYPE_ASSIGN, "assign"}, // bit operator diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index d63a9ebb48..d9f582e669 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1245,8 +1245,8 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); - _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); - _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); + _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); double *output = (double *)pOutputCol->pData; @@ -1257,17 +1257,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam continue; } - int64_t lx = getVectorBigintValueFnLeft(LEFT_COL, i); - int64_t rx = getVectorBigintValueFnRight(RIGHT_COL, i); - if (rx == 0) { + double lx = getVectorDoubleValueFnLeft(LEFT_COL, i); + double rx = getVectorDoubleValueFnRight(RIGHT_COL, i); + if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) { colDataAppendNULL(pOutputCol, i); continue; } - *output = lx % rx; + *output = lx - ((int64_t)(lx / rx)) * rx; } } else if (pLeft->numOfRows == 1) { - int64_t lx = getVectorBigintValueFnLeft(LEFT_COL, 0); + double lx = getVectorDoubleValueFnLeft(LEFT_COL, 0); if (IS_HELPER_NULL(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); } else { @@ -1277,18 +1277,18 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam continue; } - int64_t rx = getVectorBigintValueFnRight(RIGHT_COL, i); - if (rx == 0){ + double rx = getVectorDoubleValueFnRight(RIGHT_COL, i); + if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) { colDataAppendNULL(pOutputCol, i); continue; } - *output = lx % rx; + *output = lx - ((int64_t)(lx / rx)) * rx; } } } else if (pRight->numOfRows == 1) { - int64_t rx = getVectorBigintValueFnRight(RIGHT_COL, 0); - if (IS_HELPER_NULL(pRightCol, 0) || rx == 0) { // Set pLeft->numOfRows NULL value + double rx = getVectorDoubleValueFnRight(RIGHT_COL, 0); + if (IS_HELPER_NULL(pRightCol, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); } else { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { @@ -1297,8 +1297,13 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam continue; } - int64_t lx = getVectorBigintValueFnLeft(LEFT_COL, i); - *output = lx % rx; + double lx = getVectorDoubleValueFnLeft(LEFT_COL, i); + if (isnan(lx) || isinf(lx)) { + colDataAppendNULL(pOutputCol, i); + continue; + } + + *output = lx - ((int64_t)(lx / rx)) * rx; } } } @@ -1712,7 +1717,7 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { return vectorMathMultiply; case OP_TYPE_DIV: return vectorMathDivide; - case OP_TYPE_MOD: + case OP_TYPE_REM: return vectorMathRemainder; case OP_TYPE_MINUS: return vectorMathMinus; diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 2912fce8be..8827c2f415 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -1061,7 +1061,7 @@ void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){ void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32_t rightType, void *rightData, bool isReverse){ int32_t resType = TSDB_DATA_TYPE_NULL; if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || - opType == OP_TYPE_DIV || opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){ + opType == OP_TYPE_DIV || opType == OP_TYPE_REM || opType == OP_TYPE_MINUS){ resType = TSDB_DATA_TYPE_DOUBLE; }else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){ resType = TSDB_DATA_TYPE_BIGINT; @@ -1106,7 +1106,7 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do printf("result:NULL\n"); }else if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || opType == OP_TYPE_DIV || - opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){ + opType == OP_TYPE_REM || opType == OP_TYPE_MINUS){ printf("op:%s,1result:%f,except:%f\n", gOptrStr[opType].str, *((double *)colDataGetData(column, 0)), exceptValue); ASSERT_TRUE(fabs(*((double *)colDataGetData(column, 0)) - exceptValue) < 0.0001); }else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){ @@ -1136,7 +1136,7 @@ TEST(columnTest, json_column_arith_op) { const int32_t len = 8; EOperatorType op[len] = {OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV, - OP_TYPE_MOD, OP_TYPE_MINUS, OP_TYPE_BIT_AND, OP_TYPE_BIT_OR}; + OP_TYPE_REM, OP_TYPE_MINUS, OP_TYPE_BIT_AND, OP_TYPE_BIT_OR}; int32_t input[len] = {1, 8, 2, 2, 3, 0, -4, 9}; printf("--------------------json int-4 op {1, 8, 2, 2, 3, 0, -4, 9}--------------------\n"); From 73dfefdbbc75a69f800ee30dd9a646a65b04c9f0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 5 Jun 2022 22:31:25 +0800 Subject: [PATCH 61/63] feat: test arith/logic operator for json --- source/libs/scalar/test/scalar/scalarTests.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 8827c2f415..284ce4b95c 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -1189,7 +1189,7 @@ TEST(columnTest, json_column_arith_op) { printf("----------------------json double-- 5.44 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n"); key = "k5"; - double eRes40[len] = {6.44, -2.56, 10.88, 2.72, 2, -5.44, 5&-4, 5|9}; + double eRes40[len] = {6.44, -2.56, 10.88, 2.72, 2.44, -5.44, 5&-4, 5|9}; double eRes41[len] = {6.44, 2.56, 10.88, 0.3676470588235294, 3, 0, 5&-4, 5|9}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes40[i], op[i], false); @@ -1213,8 +1213,8 @@ TEST(columnTest, json_column_arith_op) { printf("----------------------json double-- -9.8 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n"); key = "k7"; - double eRes60[len] = {-8.8, -17.8, -19.6, -4.9, -9%3, 9.8, -9&-4, -9|9}; - double eRes61[len] = {-8.8, 17.8, -19.6, -0.2040816326530612, 3%-9, 0, -9&-4, -9|9}; + double eRes60[len] = {-8.8, -17.8, -19.6, -4.9, -0.8, 9.8, -9&-4, -9|9}; + double eRes61[len] = {-8.8, 17.8, -19.6, -0.2040816326530612, 3, 0, -9&-4, -9|9}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes60[i], op[i], false); } From 72657a3f60e8953dcf03e28f02cae694b92b639d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 5 Jun 2022 23:01:48 +0800 Subject: [PATCH 62/63] fix invalid conn --- source/libs/transport/src/transSvr.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 82d2b7a6db..50f99128b2 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1160,6 +1160,10 @@ int transGetConnInfo(void* thandle, STransHandleInfo* pInfo) { } SExHandle* ex = thandle; SSvrConn* pConn = ex->handle; + if (pConn == NULL) { + tTrace("invalid handle %p, failed to Get Conn info", thandle); + return -1; + } struct sockaddr_in addr = pConn->addr; pInfo->clientIp = (uint32_t)(addr.sin_addr.s_addr); From 64de7ee1ac1c824e40bcae5798f653d2f8ad69f7 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 6 Jun 2022 09:12:54 +0800 Subject: [PATCH 63/63] disable hb cleanup --- source/libs/scheduler/src/schUtil.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index a91dc359f2..18398802db 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -40,6 +40,8 @@ void schCleanClusterHb(void* pTrans) { } int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId) { + return TSDB_CODE_SUCCESS; // TODO ENABLE IT WHEN RPC IS READY + int32_t code = 0; SCH_LOCK(SCH_WRITE, &schMgmt.hbLock);