From 45e8ec7470b4da62dbaec6dcfd20ad75fb6b5e86 Mon Sep 17 00:00:00 2001 From: dapan Date: Mon, 9 May 2022 21:13:29 +0800 Subject: [PATCH 01/94] support no fqdn for single dnode --- include/common/tmsg.h | 1 + source/client/src/clientMsgHandler.c | 2 +- source/common/src/tmsg.c | 2 ++ source/dnode/mnode/impl/src/mndProfile.c | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9ec80293a8..394e1e0eca 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -369,6 +369,7 @@ typedef struct { int32_t acctId; int64_t clusterId; uint32_t connId; + int32_t dnodeNum; int8_t superUser; int8_t connType; SEpSet epSet; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 8096ce395a..11c6971e3d 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -58,7 +58,7 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { return code; } - if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) { + if (connectRsp.dnodeNum > 1 && !isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) { updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &connectRsp.epSet); } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 85b55611dc..9d29c652e0 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2751,6 +2751,7 @@ int32_t tSerializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { if (tEncodeI32(&encoder, pRsp->acctId) < 0) return -1; if (tEncodeI64(&encoder, pRsp->clusterId) < 0) return -1; if (tEncodeU32(&encoder, pRsp->connId) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->dnodeNum) < 0) return -1; if (tEncodeI8(&encoder, pRsp->superUser) < 0) return -1; if (tEncodeI8(&encoder, pRsp->connType) < 0) return -1; if (tEncodeSEpSet(&encoder, &pRsp->epSet) < 0) return -1; @@ -2770,6 +2771,7 @@ int32_t tDeserializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { if (tDecodeI32(&decoder, &pRsp->acctId) < 0) return -1; if (tDecodeI64(&decoder, &pRsp->clusterId) < 0) return -1; if (tDecodeU32(&decoder, &pRsp->connId) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->dnodeNum) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->superUser) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->connType) < 0) return -1; if (tDecodeSEpSet(&decoder, &pRsp->epSet) < 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 2de337537f..fbca63e3f6 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -227,6 +227,7 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) { connectRsp.clusterId = pMnode->clusterId; connectRsp.connId = pConn->id; connectRsp.connType = connReq.connType; + connectRsp.dnodeNum = mndGetDnodeSize(pMnode); snprintf(connectRsp.sVersion, sizeof(connectRsp.sVersion), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, gitinfo); From 0058775309c039c7b220b24f23a555ceb3351620 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 10 May 2022 05:57:31 +0000 Subject: [PATCH 02/94] feat: TD-15433 --- include/common/tmsg.h | 30 +++++++---- source/common/src/tmsg.c | 62 ++++++++++++++++++++++ source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 5 +- source/dnode/vnode/src/tsdb/tsdbWrite.c | 7 +-- source/dnode/vnode/src/vnd/vnodeSvr.c | 43 +++++++++++---- source/libs/scheduler/src/scheduler.c | 2 +- 7 files changed, 122 insertions(+), 29 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9ec80293a8..e6aaff211b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -252,21 +252,29 @@ STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter); int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq* pReq, STSchema* pSchema); typedef struct { - int32_t index; // index of failed block in submit blocks - int32_t vnode; // vnode index of failed block - int32_t sid; // table index of failed block - int32_t code; // errorcode while write data to vnode, such as not created, dropped, no space, invalid table -} SSubmitRspBlock; + int8_t hashMeta; + int64_t uid; + union { + char* name; + const char* namec; + }; + int32_t numOfRows; + int32_t affectedRows; +} SSubmitBlkRsp; typedef struct { - int32_t code; // 0-success, > 0 error code - int32_t numOfRows; // number of records the client is trying to write - int32_t affectedRows; // number of records actually written - int32_t failedRows; // number of failed records (exclude duplicate records) - int32_t numOfFailedBlocks; - SSubmitRspBlock failedBlocks[]; + int32_t numOfRows; + int32_t affectedRows; + int32_t nBlocks; + union { + SArray* pArray; + SSubmitBlkRsp* pBlocks; + }; } SSubmitRsp; +int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); +int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); + #define SCHEMA_SMA_ON 0x1 #define SCHEMA_IDX_ON 0x2 typedef struct SSchema { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 85b55611dc..679ec5892d 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4013,3 +4013,65 @@ int32_t tDecodeSVSubmitReq(SDecoder *pCoder, SVSubmitReq *pReq) { tEndDecode(pCoder); return 0; } + +static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBlock) { + if (tStartEncode(pEncoder) < 0) return -1; + + if (tEncodeI8(pEncoder, pBlock->hashMeta) < 0) return -1; + if (pBlock->hashMeta) { + if (tEncodeI64(pEncoder, pBlock->uid) < 0) return -1; + if (tEncodeCStr(pEncoder, pBlock->name) < 0) return -1; + } + if (tEncodeI32v(pEncoder, pBlock->numOfRows) < 0) return -1; + if (tEncodeI32v(pEncoder, pBlock->affectedRows) < 0) return -1; + + tEndEncode(pEncoder); + return 0; +} + +static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { + if (tStartDecode(pDecoder) < 0) return -1; + + if (tDecodeI8(pDecoder, &pBlock->hashMeta) < 0) return -1; + if (pBlock->hashMeta) { + if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; + if (tDecodeCStr(pDecoder, &pBlock->namec) < 0) return -1; + } + if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1; + if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1; + + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeSSubmitRsp(SEncoder *pEncoder, const SSubmitRsp *pRsp) { + int32_t nBlocks = taosArrayGetSize(pRsp->pArray); + + if (tStartEncode(pEncoder) < 0) return -1; + + if (tEncodeI32v(pEncoder, pRsp->numOfRows) < 0) return -1; + if (tEncodeI32v(pEncoder, pRsp->affectedRows) < 0) return -1; + if (tEncodeI32v(pEncoder, nBlocks) < 0) return -1; + for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) { + if (tEncodeSSubmitBlkRsp(pEncoder, (SSubmitBlkRsp *)taosArrayGet(pRsp->pArray, iBlock)) < 0) return -1; + } + + tEndEncode(pEncoder); + return 0; +} + +int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) { + if (tStartDecode(pDecoder) < 0) return -1; + + if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1; + if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1; + if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1; + pRsp->pBlocks = tDecoderMalloc(pDecoder, sizeof(*pRsp->pBlocks) * pRsp->nBlocks); + if (pRsp->pBlocks == NULL) return -1; + for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) { + if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1; + } + + tEndDecode(pDecoder); + return 0; +} diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 986b2740f3..9a36fc6eae 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -102,7 +102,7 @@ int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg); int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); -int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, int32_t* pAffectedRows); +int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp); tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId); tsdbReaderT tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index d514512881..95156c1e1c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -288,7 +288,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey return 0; } -int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, int32_t *pAffectedRows) { +int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkRsp *pRsp) { SSubmitBlkIter blkIter = {0}; STsdbMemTable *pMemTable = pTsdb->mem; void *tptr; @@ -342,7 +342,8 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo if (pMemTable->keyMin > keyMin) pMemTable->keyMin = keyMin; if (pMemTable->keyMax < keyMax) pMemTable->keyMax = keyMax; - (*pAffectedRows) = pMsgIter->numOfRows; + pRsp->numOfRows = pRsp->numOfRows; + pRsp->affectedRows = pRsp->affectedRows; return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 5a7892a750..3107c6f5c7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -36,9 +36,10 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * // loop to insert tInitSubmitMsgIter(pMsg, &msgIter); while (true) { + SSubmitBlkRsp r = {0}; tGetSubmitMsgNext(&msgIter, &pBlock); if (pBlock == NULL) break; - if (tsdbInsertTableData(pTsdb, &msgIter, pBlock, &affectedrows) < 0) { + if (tsdbInsertTableData(pTsdb, &msgIter, pBlock, &r) < 0) { return -1; } @@ -46,8 +47,8 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * } if (pRsp != NULL) { - pRsp->affectedRows = affectedrows; - pRsp->numOfRows = numOfRows; + // pRsp->affectedRows = affectedrows; + // pRsp->numOfRows = numOfRows; } return 0; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 0082ca0802..7692b7084d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -497,7 +497,7 @@ _exit: return 0; } -static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char* tags) { +static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char *tags) { ASSERT(pMsg != NULL); SSubmitMsgIter msgIter = {0}; SMeta *pMeta = pVnode->pMeta; @@ -518,11 +518,11 @@ static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char taosMemoryFreeClear(pSchema); } pSchema = metaGetTbTSchema(pMeta, msgIter.suid, 0); // TODO: use the real schema - if(pSchema) { + if (pSchema) { suid = msgIter.suid; } } - if(!pSchema) { + if (!pSchema) { printf("%s:%d no valid schema\n", tags, __LINE__); continue; } @@ -540,12 +540,15 @@ static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + SSubmitRsp submitRsp = {0}; SSubmitMsgIter msgIter = {0}; SSubmitBlk *pBlock; SSubmitRsp rsp = {0}; SVCreateTbReq createTbReq = {0}; SDecoder decoder = {0}; int32_t nRows; + int32_t tsize, ret; + SEncoder encoder = {0}; pRsp->code = 0; @@ -559,12 +562,17 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in goto _exit; } - for (;;) { + submitRsp.pArray = taosArrayInit(pSubmitReq->numOfBlocks, sizeof(SSubmitBlkRsp)); + for (int i = 0;;) { tGetSubmitMsgNext(&msgIter, &pBlock); if (pBlock == NULL) break; + SSubmitBlkRsp submitBlkRsp = {0}; + // create table for auto create table mode if (msgIter.schemaLen > 0) { + submitBlkRsp.hashMeta = 1; + tDecoderInit(&decoder, pBlock->data, msgIter.schemaLen); if (tDecodeSVCreateTbReq(&decoder, &createTbReq) < 0) { pRsp->code = TSDB_CODE_INVALID_MSG; @@ -580,6 +588,10 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in } } + submitBlkRsp.uid = createTbReq.uid; + submitBlkRsp.name = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2); + sprintf(submitBlkRsp.name, "%s.%s", pVnode->config.dbname, createTbReq.name); + msgIter.uid = createTbReq.uid; if (createTbReq.type == TSDB_CHILD_TABLE) { msgIter.suid = createTbReq.ctb.suid; @@ -590,20 +602,29 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in tDecoderClear(&decoder); } - if (tsdbInsertTableData(pVnode->pTsdb, &msgIter, pBlock, &nRows) < 0) { + if (tsdbInsertTableData(pVnode->pTsdb, &msgIter, pBlock, &submitBlkRsp) < 0) { pRsp->code = terrno; goto _exit; } - rsp.affectedRows += nRows; - + submitRsp.numOfRows += submitBlkRsp.numOfRows; + submitRsp.affectedRows += submitBlkRsp.affectedRows; + taosArrayPush(submitRsp.pArray, &submitBlkRsp); } _exit: - // encode the response (TODO) - pRsp->pCont = rpcMallocCont(sizeof(SSubmitRsp)); - memcpy(pRsp->pCont, &rsp, sizeof(rsp)); - pRsp->contLen = sizeof(SSubmitRsp); + tEncodeSize(tEncodeSSubmitRsp, &submitRsp, tsize, ret); + pRsp->pCont = rpcMallocCont(tsize); + pRsp->contLen = tsize; + tEncoderInit(&encoder, pRsp->pCont, tsize); + tEncodeSSubmitRsp(&encoder, &submitRsp); + tEncoderClear(&encoder); + + for (int32_t i = 0; i < taosArrayGetSize(submitRsp.pArray); i++) { + taosMemoryFree(((SSubmitBlkRsp *)taosArrayGet(submitRsp.pArray, i))[0].name); + } + + taosArrayDestroy(submitRsp.pArray); tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 10e4255022..a8b0f2e2b8 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1135,7 +1135,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch case TDMT_VND_SUBMIT_RSP: { if (msg) { SSubmitRsp *rsp = (SSubmitRsp *)msg; - SCH_ERR_JRET(rsp->code); + // SCH_ERR_JRET(rsp->code); } SCH_ERR_JRET(rspCode); From 607d5c451920032f93abea6e2482c99f5c7d1d3a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 10 May 2022 06:07:03 +0000 Subject: [PATCH 03/94] refact --- include/common/tmsg.h | 4 ++-- source/common/src/tmsg.c | 4 ++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index e6aaff211b..73d5c4fd90 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -255,8 +255,8 @@ typedef struct { int8_t hashMeta; int64_t uid; union { - char* name; - const char* namec; + char* ename; // used for encode + const char* dname; // used for decode }; int32_t numOfRows; int32_t affectedRows; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 679ec5892d..8656ee3eb0 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4020,7 +4020,7 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl if (tEncodeI8(pEncoder, pBlock->hashMeta) < 0) return -1; if (pBlock->hashMeta) { if (tEncodeI64(pEncoder, pBlock->uid) < 0) return -1; - if (tEncodeCStr(pEncoder, pBlock->name) < 0) return -1; + if (tEncodeCStr(pEncoder, pBlock->ename) < 0) return -1; } if (tEncodeI32v(pEncoder, pBlock->numOfRows) < 0) return -1; if (tEncodeI32v(pEncoder, pBlock->affectedRows) < 0) return -1; @@ -4035,7 +4035,7 @@ static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { if (tDecodeI8(pDecoder, &pBlock->hashMeta) < 0) return -1; if (pBlock->hashMeta) { if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; - if (tDecodeCStr(pDecoder, &pBlock->namec) < 0) return -1; + if (tDecodeCStr(pDecoder, &pBlock->dname) < 0) return -1; } if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1; if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 7692b7084d..678c9edbd4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -589,8 +589,8 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in } submitBlkRsp.uid = createTbReq.uid; - submitBlkRsp.name = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2); - sprintf(submitBlkRsp.name, "%s.%s", pVnode->config.dbname, createTbReq.name); + submitBlkRsp.ename = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2); + sprintf(submitBlkRsp.ename, "%s.%s", pVnode->config.dbname, createTbReq.name); msgIter.uid = createTbReq.uid; if (createTbReq.type == TSDB_CHILD_TABLE) { @@ -621,7 +621,7 @@ _exit: tEncoderClear(&encoder); for (int32_t i = 0; i < taosArrayGetSize(submitRsp.pArray); i++) { - taosMemoryFree(((SSubmitBlkRsp *)taosArrayGet(submitRsp.pArray, i))[0].name); + taosMemoryFree(((SSubmitBlkRsp *)taosArrayGet(submitRsp.pArray, i))[0].ename); } taosArrayDestroy(submitRsp.pArray); From 005ff0eda912e925616c254d2e0542854f9297d8 Mon Sep 17 00:00:00 2001 From: dapan Date: Tue, 10 May 2022 15:50:41 +0800 Subject: [PATCH 04/94] stmt auto create table --- include/common/taosdef.h | 2 +- include/libs/parser/parser.h | 6 +- source/client/inc/clientStmt.h | 5 +- source/client/src/clientStmt.c | 81 +++++++++++++++--------- source/dnode/mnode/impl/src/mndProfile.c | 1 + source/libs/parser/inc/parInsertData.h | 2 +- source/libs/parser/src/parInsert.c | 60 ++++++++++-------- source/libs/parser/src/parInsertData.c | 18 ++++-- source/util/src/tuuid.c | 16 +++-- tests/script/api/batchprepare.c | 2 +- 10 files changed, 117 insertions(+), 76 deletions(-) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 5384082da3..e1f8832edf 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -89,7 +89,7 @@ extern char *qtypeStr[]; #define TSDB_PORT_HTTP 11 -#define TD_DEBUG_PRINT_ROW +#undef TD_DEBUG_PRINT_ROW #ifdef __cplusplus } diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 998d45aee1..eab834285e 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -26,8 +26,7 @@ extern "C" { typedef struct SStmtCallback { TAOS_STMT* pStmt; int32_t (*getTbNameFn)(TAOS_STMT*, char**); - int32_t (*setBindInfoFn)(TAOS_STMT*, STableMeta*, void*); - int32_t (*setExecInfoFn)(TAOS_STMT*, SHashObj*, SHashObj*); + int32_t (*setInfoFn)(TAOS_STMT*, STableMeta*, void*, char*, bool, SHashObj*, SHashObj*); int32_t (*getExecInfoFn)(TAOS_STMT*, SHashObj**, SHashObj**); } SStmtCallback; @@ -89,8 +88,9 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash int32_t qResetStmtDataBlock(void* block, bool keepBuf); int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); void qFreeStmtDataBlock(void* pDataBlock); -int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc); +int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid); void qDestroyStmtDataBlock(void* pBlock); +STableMeta *qGetTableMetaInDataBlock(void* pDataBlock); int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 32da75fb1e..153cb84986 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -62,7 +62,8 @@ typedef struct SStmtBindInfo { int8_t tbType; bool tagsCached; void* boundTags; - char* tbName; + char tbName[TSDB_TABLE_FNAME_LEN];; + char tbFName[TSDB_TABLE_FNAME_LEN]; SName sname; } SStmtBindInfo; @@ -76,7 +77,7 @@ typedef struct SStmtExecInfo { typedef struct SStmtSQLInfo { STMT_TYPE type; STMT_STATUS status; - bool autoCreate; + bool autoCreateTbl; uint64_t runTimes; SHashObj* pTableCache; //SHash SQuery* pQuery; diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index c804213d89..61a850802a 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -68,7 +68,7 @@ int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) { pStmt->sql.type = STMT_TYPE_MULTI_INSERT; - if (NULL == pStmt->bInfo.tbName) { + if ('\0' == pStmt->bInfo.tbName[0]) { tscError("no table name set"); STMT_ERR_RET(TSDB_CODE_TSC_STMT_TBNAME_ERROR); } @@ -121,9 +121,12 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } -int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) { +int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, char* tbFName) { STscStmt* pStmt = (STscStmt*)stmt; + strncpy(pStmt->bInfo.tbFName, tbFName, sizeof(pStmt->bInfo.tbFName) - 1); + pStmt->bInfo.tbFName[sizeof(pStmt->bInfo.tbFName) - 1] = 0; + pStmt->bInfo.tbUid = pTableMeta->uid; pStmt->bInfo.tbSuid = pTableMeta->suid; pStmt->bInfo.tbType = pTableMeta->tableType; @@ -133,7 +136,7 @@ int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) { return TSDB_CODE_SUCCESS; } -int32_t stmtSetExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash) { +int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash) { STscStmt* pStmt = (STscStmt*)stmt; pStmt->exec.pVgHash = pVgHash; @@ -142,6 +145,18 @@ int32_t stmtSetExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash return TSDB_CODE_SUCCESS; } +int32_t stmtUpdateInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, char* tbFName, bool autoCreateTbl, SHashObj* pVgHash, SHashObj* pBlockHash) { + STscStmt* pStmt = (STscStmt*)stmt; + + STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbFName)); + STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash)); + + pStmt->sql.autoCreateTbl = autoCreateTbl; + + return TSDB_CODE_SUCCESS; +} + + int32_t stmtGetExecInfo(TAOS_STMT* stmt, SHashObj** pVgHash, SHashObj** pBlockHash) { STscStmt* pStmt = (STscStmt*)stmt; @@ -157,13 +172,13 @@ int32_t stmtCacheBlock(STscStmt *pStmt) { } uint64_t uid = pStmt->bInfo.tbUid; - uint64_t tuid = (TSDB_CHILD_TABLE == pStmt->bInfo.tbType) ? pStmt->bInfo.tbSuid : uid; + uint64_t cacheUid = (TSDB_CHILD_TABLE == pStmt->bInfo.tbType) ? pStmt->bInfo.tbSuid : uid; - if (taosHashGet(pStmt->sql.pTableCache, &tuid, sizeof(tuid))) { + if (taosHashGet(pStmt->sql.pTableCache, &cacheUid, sizeof(cacheUid))) { return TSDB_CODE_SUCCESS; } - STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, &uid, sizeof(uid)); + STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); STableDataBlocks* pDst = NULL; STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc)); @@ -173,7 +188,7 @@ int32_t stmtCacheBlock(STscStmt *pStmt) { .boundTags = pStmt->bInfo.boundTags, }; - if (taosHashPut(pStmt->sql.pTableCache, &tuid, sizeof(tuid), &cache, sizeof(cache))) { + if (taosHashPut(pStmt->sql.pTableCache, &cacheUid, sizeof(cacheUid), &cache, sizeof(cache))) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -186,8 +201,7 @@ int32_t stmtParseSql(STscStmt* pStmt) { SStmtCallback stmtCb = { .pStmt = pStmt, .getTbNameFn = stmtGetTbName, - .setBindInfoFn = stmtSetBindInfo, - .setExecInfoFn = stmtSetExecInfo, + .setInfoFn = stmtUpdateInfo, .getExecInfoFn = stmtGetExecInfo, }; @@ -222,7 +236,8 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) { pStmt->bInfo.tbType = 0; pStmt->bInfo.needParse = true; - taosMemoryFreeClear(pStmt->bInfo.tbName); + pStmt->bInfo.tbName[0] = 0; + pStmt->bInfo.tbFName[0] = 0; if (!pStmt->bInfo.tagsCached) { destroyBoundColumnInfo(pStmt->bInfo.boundTags); taosMemoryFreeClear(pStmt->bInfo.boundTags); @@ -237,12 +252,14 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) { pStmt->exec.pRequest = NULL; } + size_t keyLen = 0; void *pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { STableDataBlocks* pBlocks = *(STableDataBlocks**)pIter; - uint64_t *key = taosHashGetKey(pIter, NULL); + char *key = taosHashGetKey(pIter, &keyLen); + STableMeta* pMeta = qGetTableMetaInDataBlock(pBlocks); - if (keepTable && (*key == pStmt->bInfo.tbUid)) { + if (keepTable && (pMeta->uid == pStmt->bInfo.tbUid)) { STMT_ERR_RET(qResetStmtDataBlock(pBlocks, true)); pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); @@ -250,7 +267,7 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) { } qFreeStmtDataBlock(pBlocks); - taosHashRemove(pStmt->exec.pBlockHash, key, sizeof(*key)); + taosHashRemove(pStmt->exec.pBlockHash, key, keyLen); pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); } @@ -322,6 +339,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { uint64_t suid = pTableMeta->suid; int8_t tableType = pTableMeta->tableType; taosMemoryFree(pTableMeta); + uint64_t cacheUid = (TSDB_CHILD_TABLE == tableType) ? suid : uid; if (uid == pStmt->bInfo.tbUid) { pStmt->bInfo.needParse = false; @@ -329,10 +347,10 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } - if (taosHashGet(pStmt->exec.pBlockHash, &uid, sizeof(uid))) { - SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid)); + if (taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName))) { + SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &cacheUid, sizeof(cacheUid)); if (NULL == pCache) { - tscError("table uid %" PRIx64 "found in exec blockHash, but not in sql blockHash", uid); + tscError("table [%s, %" PRIx64 ", %" PRIx64 "] found in exec blockHash, but not in sql blockHash", pStmt->bInfo.tbFName, uid, cacheUid); STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); } @@ -348,7 +366,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } - SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid)); + SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &cacheUid, sizeof(cacheUid)); if (pCache) { pStmt->bInfo.needParse = false; @@ -359,9 +377,9 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.tagsCached = true; STableDataBlocks* pNewBlock = NULL; - STMT_ERR_RET(qRebuildStmtDataBlock(&pNewBlock, pCache->pDataBlock)); + STMT_ERR_RET(qRebuildStmtDataBlock(&pNewBlock, pCache->pDataBlock, uid)); - if (taosHashPut(pStmt->exec.pBlockHash, &pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid), &pNewBlock, POINTER_BYTES)) { + if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -449,12 +467,13 @@ int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) { } STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); - + tNameExtractFullName(&pStmt->bInfo.sname, pStmt->bInfo.tbFName); + STMT_ERR_RET(stmtGetFromCache(pStmt)); if (pStmt->bInfo.needParse) { - taosMemoryFree(pStmt->bInfo.tbName); - pStmt->bInfo.tbName = strdup(tbName); + strncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName) - 1); + pStmt->bInfo.tbName[sizeof(pStmt->bInfo.tbName) - 1] = 0; } return TSDB_CODE_SUCCESS; @@ -465,15 +484,15 @@ int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS)); - if (!pStmt->bInfo.needParse) { + if (!pStmt->bInfo.needParse) { // table already cached, no need create table return TSDB_CODE_SUCCESS; } STMT_ERR_RET(stmtParseSql(pStmt)); - STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); + STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { - tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); + tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } @@ -489,9 +508,9 @@ int32_t stmtFetchTagFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fiel STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); + STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { - tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); + tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } @@ -506,9 +525,9 @@ int32_t stmtFetchColFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fiel STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); + STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { - tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); + tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } @@ -552,9 +571,9 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) { STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx, pStmt->exec.pRequest->requestId)); } - STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); + STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { - tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); + tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index fbca63e3f6..600bdcf310 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -20,6 +20,7 @@ #include "mndShow.h" #include "mndStb.h" #include "mndUser.h" +#include "mndDnode.h" #include "tglobal.h" #include "version.h" diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index 0627ed1066..e19f54dff3 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -137,7 +137,7 @@ void destroyBlockArrayList(SArray* pDataBlockList); void destroyBlockHashmap(SHashObj* pDataBlockHash); int initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo); int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); -int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, +int32_t getDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset, int32_t rowSize, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq); int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** pVgDataBlocks); int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 64d2934282..f1febb47cd 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -237,13 +237,8 @@ static int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, con return code; } -static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SToken* pTname, bool isStb) { +static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SName* name, char *dbFname, bool isStb) { SParseContext* pBasicCtx = pCxt->pComCxt; - SName name = {0}; - createSName(&name, pTname, pBasicCtx->acctId, pBasicCtx->db, &pCxt->msg); - - char dbFname[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(&name, dbFname); bool pass = false; CHECK_CODE(catalogChkAuth(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, pBasicCtx->pUser, dbFname, AUTH_TYPE_WRITE, &pass)); @@ -251,22 +246,22 @@ static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SToken* pTname, bool return TSDB_CODE_PAR_PERMISSION_DENIED; } if (isStb) { - CHECK_CODE(catalogGetSTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, + CHECK_CODE(catalogGetSTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, name, &pCxt->pTableMeta)); } else { - CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, + CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, name, &pCxt->pTableMeta)); SVgroupInfo vg; CHECK_CODE( - catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg)); + catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, name, &vg)); CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); } return TSDB_CODE_SUCCESS; } -static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { return getTableMetaImpl(pCxt, pTname, false); } +static int32_t getTableMeta(SInsertParseContext* pCxt, SName* name, char *dbFname) { return getTableMetaImpl(pCxt, name, dbFname, false); } -static int32_t getSTableMeta(SInsertParseContext* pCxt, SToken* pTname) { return getTableMetaImpl(pCxt, pTname, true); } +static int32_t getSTableMeta(SInsertParseContext* pCxt, SName* name, char *dbFname) { return getTableMetaImpl(pCxt, name, dbFname, true); } static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchema) { while (start < end) { @@ -842,7 +837,7 @@ static int32_t storeTableMeta(SInsertParseContext* pCxt, SHashObj* pHash, SName* catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, pTableName, &vg)); CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); - pMeta->uid = tGenIdPI64(); + pMeta->uid = 0; pMeta->vgId = vg.vgId; STableMeta* pBackup = NULL; @@ -853,11 +848,7 @@ static int32_t storeTableMeta(SInsertParseContext* pCxt, SHashObj* pHash, SName* } // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) -static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) { - SName name; - createSName(&name, pTbnameToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); - char tbFName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(&name, tbFName); +static int32_t parseUsingClause(SInsertParseContext* pCxt, SName* name, char* tbFName) { int32_t len = strlen(tbFName); STableMeta** pMeta = taosHashGet(pCxt->pSubTableHashObj, tbFName, len); if (NULL != pMeta) { @@ -867,11 +858,17 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) SToken sToken; // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) NEXT_TOKEN(pCxt->pSql, sToken); - CHECK_CODE(getSTableMeta(pCxt, &sToken)); + + SName sname; + createSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); + char stbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&sname, stbFName); + + CHECK_CODE(getSTableMeta(pCxt, &sname, stbFName)); if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) { return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); } - CHECK_CODE(storeTableMeta(pCxt, pCxt->pSubTableHashObj, &name, tbFName, len, pCxt->pTableMeta)); + CHECK_CODE(storeTableMeta(pCxt, pCxt->pSubTableHashObj, name, tbFName, len, pCxt->pTableMeta)); SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta); setBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); @@ -891,7 +888,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) if (TK_NK_LP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); } - CHECK_CODE(parseTagsClause(pCxt, pCxt->pTableMeta->schema, getTableInfo(pCxt->pTableMeta).precision, name.tname)); + CHECK_CODE(parseTagsClause(pCxt, pCxt->pTableMeta->schema, getTableInfo(pCxt->pTableMeta).precision, name->tname)); NEXT_TOKEN(pCxt->pSql, sToken); if (TK_NK_RP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, ") is expected", sToken.z); @@ -1066,6 +1063,8 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) { // [...]; static int32_t parseInsertBody(SInsertParseContext* pCxt) { int32_t tbNum = 0; + char tbFName[TSDB_TABLE_FNAME_LEN]; + bool autoCreateTbl = false; // for each table while (1) { @@ -1095,6 +1094,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { sToken.z = tbName; sToken.n = strlen(tbName); + + autoCreateTbl = true; } else { return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); } @@ -1103,16 +1104,20 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { SToken tbnameToken = sToken; NEXT_TOKEN(pCxt->pSql, sToken); + SName name; + createSName(&name, &tbnameToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); + tNameExtractFullName(&name, tbFName); + // USING cluase if (TK_USING == sToken.type) { - CHECK_CODE(parseUsingClause(pCxt, &tbnameToken)); + CHECK_CODE(parseUsingClause(pCxt, &name, tbFName)); NEXT_TOKEN(pCxt->pSql, sToken); } else { - CHECK_CODE(getTableMeta(pCxt, &tbnameToken)); + CHECK_CODE(getTableMeta(pCxt, &name, tbFName)); } STableDataBlocks* dataBuf = NULL; - CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, + CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, tbFName, strlen(tbFName), TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL, &pCxt->createTblReq)); @@ -1154,10 +1159,9 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } memcpy(tags, &pCxt->tags, sizeof(pCxt->tags)); - (*pCxt->pStmtCb->setBindInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pTableMeta, tags); - memset(&pCxt->tags, 0, sizeof(pCxt->tags)); + (*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pTableMeta, tags, tbFName, autoCreateTbl, pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj); - (*pCxt->pStmtCb->setExecInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj); + memset(&pCxt->tags, 0, sizeof(pCxt->tags)); pCxt->pVgroupsHashObj = NULL; pCxt->pTableBlockHashObj = NULL; pCxt->pTableMeta = NULL; @@ -1194,7 +1198,7 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { &context.pTableBlockHashObj); } else { context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - context.pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + context.pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); } if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || NULL == context.pSubTableHashObj || @@ -1673,7 +1677,7 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SArray *cols buildCreateTbReq(&smlHandle->createTblReq, tableName, row, pTableMeta->suid); STableDataBlocks* pDataBlock = NULL; - ret = getDataBlockFromList(smlHandle->pBlockHash, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, + ret = getDataBlockFromList(smlHandle->pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid), TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, pTableMeta, &pDataBlock, NULL, &smlHandle->createTblReq); if(ret != TSDB_CODE_SUCCESS){ diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index c074334722..8bf1fe012e 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -185,11 +185,11 @@ int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) return TSDB_CODE_SUCCESS; } -int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, +int32_t getDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset, int32_t rowSize, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { *dataBlocks = NULL; - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); + STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)id, idLen); if (t1 != NULL) { *dataBlocks = *t1; } @@ -207,7 +207,7 @@ int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int3 } } - taosHashPut(pHashList, (const char*)&id, sizeof(int64_t), (char*)dataBlocks, POINTER_BYTES); + taosHashPut(pHashList, (const char*)id, idLen, (char*)dataBlocks, POINTER_BYTES); if (pBlockList) { taosArrayPush(pBlockList, dataBlocks); } @@ -457,7 +457,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p STableDataBlocks* dataBuf = NULL; pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId int32_t ret = - getDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, + getDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, sizeof(pOneTableBlock->vgId), TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL); if (ret != TSDB_CODE_SUCCESS) { taosHashCleanup(pVnodeDataBlockHashList); @@ -620,7 +620,7 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) { return qResetStmtDataBlock(*pDst, false); } -int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc) { +int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid) { int32_t code = qCloneStmtDataBlock(pDst, pSrc); if (code) { return code; @@ -633,11 +633,19 @@ int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc) { return TSDB_CODE_OUT_OF_MEMORY; } + if (pBlock->pTableMeta) { + pBlock->pTableMeta->uid = uid; + } + memset(pBlock->pData, 0, sizeof(SSubmitBlk)); return TSDB_CODE_SUCCESS; } +STableMeta *qGetTableMetaInDataBlock(void* pDataBlock) { + return ((STableDataBlocks*)pDataBlock)->pTableMeta; +} + void qFreeStmtDataBlock(void* pDataBlock) { if (pDataBlock == NULL) { return; diff --git a/source/util/src/tuuid.c b/source/util/src/tuuid.c index 69cf7baaad..1f3cbd9573 100644 --- a/source/util/src/tuuid.c +++ b/source/util/src/tuuid.c @@ -48,10 +48,18 @@ int64_t tGenIdPI64(void) { } } - int64_t ts = taosGetTimestampMs(); - uint64_t pid = taosGetPId(); - int32_t val = atomic_add_fetch_32(&tUUIDSerialNo, 1); + int64_t id; + + while (true) { + int64_t ts = taosGetTimestampMs(); + uint64_t pid = taosGetPId(); + int32_t val = atomic_add_fetch_32(&tUUIDSerialNo, 1); + + id = ((tUUIDHashId & 0x07FF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); + if (id) { + break; + } + } - int64_t id = ((tUUIDHashId & 0x07FF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); return id; } diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 8169e6c503..5cccafa1b4 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -2988,7 +2988,7 @@ void prepareCheckResultImpl(TAOS * taos, char *tname, bool printr, int expec } } else { printf("!!!expect rows %d mis-match rows %d fetched from %s\n", expected, rows, tname); - exit(1); + //exit(1); } } From ac27d62f3316e11729fe4e06add50dd053206afb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 May 2022 02:29:53 +0000 Subject: [PATCH 05/94] refact data code --- include/common/tdataformat.h | 73 ++++++++++++++++----- include/common/tmsg.h | 7 +- source/common/src/tdataformat.c | 32 +++++++-- source/dnode/vnode/src/tq/tq.c | 6 +- source/libs/parser/src/parTranslater.c | 4 +- source/libs/parser/test/parInitialCTest.cpp | 2 +- 6 files changed, 92 insertions(+), 32 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 1f3b787538..5a977b22e7 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -18,6 +18,7 @@ #include "os.h" #include "talgo.h" +#include "tencode.h" #include "ttypes.h" #include "tutil.h" @@ -25,6 +26,59 @@ extern "C" { #endif +typedef struct SSchema SSchema; +typedef struct STColumn STColumn; +typedef struct STSchema STSchema; +typedef struct STSRow2 STSRow2; +typedef struct STSRowBuilder STSRowBuilder; + +#define TD_KV_ROW 0x1U + +// STSchema + +// STSRow2 +int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow); +int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow); + +// STSchema +int32_t tTSchemaCreate(STSchema **ppTSchema); +int32_t tTSchemaDestroy(STSchema *pTSchema); + +// STRUCT ================= +struct STColumn { + col_id_t colId; + int8_t type; + int8_t flags; + int32_t bytes; + int32_t offset; +}; + +struct STSchema { + int32_t numOfCols; + schema_ver_t version; + uint16_t flen; + int32_t vlen; + int32_t tlen; + STColumn columns[]; +}; + +struct STSRow2 { + TSKEY ts; + uint32_t flags; + union { + int32_t sver; + int32_t ncols; + }; + uint32_t nData; + const uint8_t *pData; +}; + +struct STSRowBuilder { + STSchema *pTSchema; + STSRow2 row; +}; + +#if 1 //==================================== // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP #define TD_SUPPORT_READ2 @@ -59,15 +113,6 @@ extern "C" { } while (0); // ----------------- TSDB COLUMN DEFINITION -#pragma pack(push, 1) -typedef struct { - col_id_t colId; // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1)) - int8_t type; // column type - int8_t flags; // flags: 0 no index, 1 SCHEMA_SMA_ON, 2 SCHEMA_IDX_ON - int32_t bytes; // column bytes (0~16M) - int32_t offset; // point offset in STpRow after the header part. -} STColumn; -#pragma pack(pop) #define colType(col) ((col)->type) #define colFlags(col) ((col)->flags) @@ -82,15 +127,6 @@ typedef struct { #define colSetOffset(col, o) (colOffset(col) = (o)) // ----------------- TSDB SCHEMA DEFINITION -typedef struct { - int32_t numOfCols; // Number of columns appended - schema_ver_t version; // schema version - uint16_t flen; // First part length in a STpRow after the header part - int32_t vlen; // pure value part length, excluded the overhead (bytes only) - int32_t tlen; // maximum length of a STpRow without the header part - // (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + (bytes)) - STColumn columns[]; -} STSchema; #define schemaNCols(s) ((s)->numOfCols) #define schemaVersion(s) ((s)->version) @@ -386,6 +422,7 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co return 0; } +#endif #ifdef __cplusplus } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ff2e419c75..26bfa997f7 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -267,8 +267,9 @@ typedef struct { SSubmitRspBlock failedBlocks[]; } SSubmitRsp; -#define SCHEMA_SMA_ON 0x1 -#define SCHEMA_IDX_ON 0x2 +#define COL_SMA_ON ((int8_t)0x1) +#define COL_IDX_ON ((int8_t)0x2) +#define COL_VAL_SET ((int8_t)0x4) typedef struct SSchema { int8_t type; int8_t flags; @@ -277,7 +278,7 @@ typedef struct SSchema { char name[TSDB_COL_NAME_LEN]; } SSchema; -#define IS_BSMA_ON(s) (((s)->flags & 0x01) == SCHEMA_SMA_ON) +#define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON) #define SSCHMEA_TYPE(s) ((s)->type) #define SSCHMEA_FLAGS(s) ((s)->flags) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 5d893fe398..558190a8ab 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -19,12 +19,33 @@ #include "tdatablock.h" #include "tlog.h" +int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) { + if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1; + if (tEncodeU32v(pEncoder, pRow->flags) < 0) return -1; + if (pRow->flags & TD_KV_ROW) { + if (tEncodeI32v(pEncoder, pRow->ncols) < 0) return -1; + } else { + if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1; + } + if (tEncodeBinary(pEncoder, pRow->pData, pRow->nData) < 0) return -1; + return 0; +} + +int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) { + if (tDecodeI64(pDecoder, &pRow->ts) < 0) return -1; + if (tDecodeU32v(pDecoder, &pRow->flags) < 0) return -1; + if (pRow->flags & TD_KV_ROW) { + if (tDecodeI32v(pDecoder, &pRow->ncols) < 0) return -1; + } else { + if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1; + } + if (tDecodeBinary(pDecoder, &pRow->pData, &pRow->nData) < 0) return -1; + return 0; +} + +#if 1 // ==================== static void dataColSetNEleNull(SDataCol *pCol, int nEle); -#if 0 -static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, - int limit2, int tRows, bool forceSetNull); -#endif -int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { +int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { int spaceNeeded = pCol->bytes * maxPoints; if (IS_VAR_DATA_TYPE(pCol->type)) { spaceNeeded += sizeof(VarDataOffsetT) * maxPoints; @@ -504,3 +525,4 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { return row; } +#endif \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6ca60945cd..ef32216810 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -66,9 +66,9 @@ static void tdSRowDemo() { SRowBuilder rb = {0}; SSchema schema[DEMO_N_COLS] = { - {.type = TSDB_DATA_TYPE_TIMESTAMP, .colId = 1, .name = "ts", .bytes = 8, .flags = SCHEMA_SMA_ON}, - {.type = TSDB_DATA_TYPE_INT, .colId = 2, .name = "c1", .bytes = 4, .flags = SCHEMA_SMA_ON}, - {.type = TSDB_DATA_TYPE_INT, .colId = 3, .name = "c2", .bytes = 4, .flags = SCHEMA_SMA_ON}}; + {.type = TSDB_DATA_TYPE_TIMESTAMP, .colId = 1, .name = "ts", .bytes = 8, .flags = COL_SMA_ON}, + {.type = TSDB_DATA_TYPE_INT, .colId = 2, .name = "c1", .bytes = 4, .flags = COL_SMA_ON}, + {.type = TSDB_DATA_TYPE_INT, .colId = 3, .name = "c2", .bytes = 4, .flags = COL_SMA_ON}}; SSchema* pSchema = schema; STSchema* pTSChema = tdGetSTSChemaFromSSChema(&pSchema, numOfCols); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index cdcb2592a7..5b984d766c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2138,7 +2138,7 @@ static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray) { SField field = {.type = pCol->dataType.type, .bytes = calcTypeBytes(pCol->dataType)}; strcpy(field.name, pCol->colName); if (pCol->sma) { - field.flags |= SCHEMA_SMA_ON; + field.flags |= COL_SMA_ON; } taosArrayPush(*pArray, &field); } @@ -2321,7 +2321,7 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchema) { int8_t flags = 0; if (pCol->sma) { - flags |= SCHEMA_SMA_ON; + flags |= COL_SMA_ON; } pSchema->colId = colId; pSchema->type = pCol->dataType.type; diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index cf364aba5c..80921f59a9 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -117,7 +117,7 @@ TEST_F(ParserInitialCTest, createStable) { }; auto addFieldToCreateStbReqFunc = [&](bool col, const char* pFieldName, uint8_t type, int32_t bytes = 0, - int8_t flags = SCHEMA_SMA_ON) { + int8_t flags = COL_SMA_ON) { SField field = {0}; strcpy(field.name, pFieldName); field.type = type; From 89e8f14a791d4e6b1852bc0d3bf7c4e8a3ef3049 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 May 2022 02:37:52 +0000 Subject: [PATCH 06/94] refact: data format --- include/common/tdataformat.h | 16 ++++++++-------- source/common/src/tdataformat.c | 13 +++++++++++++ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 5a977b22e7..ee61098580 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -41,8 +41,8 @@ int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow); int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow); // STSchema -int32_t tTSchemaCreate(STSchema **ppTSchema); -int32_t tTSchemaDestroy(STSchema *pTSchema); +int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema); +void tTSchemaDestroy(STSchema *pTSchema); // STRUCT ================= struct STColumn { @@ -54,12 +54,12 @@ struct STColumn { }; struct STSchema { - int32_t numOfCols; - schema_ver_t version; - uint16_t flen; - int32_t vlen; - int32_t tlen; - STColumn columns[]; + int32_t numOfCols; + int32_t version; + int32_t flen; + int32_t vlen; + int32_t tlen; + STColumn columns[]; }; struct STSRow2 { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 558190a8ab..3f90fa36e6 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -43,6 +43,19 @@ int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) { return 0; } +int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) { + *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols); + if (*ppTSchema == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + // (*ppTSchema) + return 0; +} + +void tTSchemaDestroy(STSchema *pTSchema) { taosMemoryFree(pTSchema); } + #if 1 // ==================== static void dataColSetNEleNull(SDataCol *pCol, int nEle); int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { From 4a2d0d6a01694480c2f303da496e17e59d9cbe9a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 May 2022 03:11:52 +0000 Subject: [PATCH 07/94] refact data format --- include/common/tdataformat.h | 10 +++- source/common/src/tdataformat.c | 91 ++++++++++++++++++++++++++++++++- 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index ee61098580..0aa09d0d06 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -41,9 +41,16 @@ int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow); int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow); // STSchema -int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema); +int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); void tTSchemaDestroy(STSchema *pTSchema); +// STSRowBuilder +int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols); +int32_t tTSRowBuilderClear(STSRowBuilder *pBuilder); +int32_t tTSRowBuilderReset(STSRowBuilder *pBuilder); +int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData); +int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow); + // STRUCT ================= struct STColumn { col_id_t colId; @@ -74,6 +81,7 @@ struct STSRow2 { }; struct STSRowBuilder { + STColumn *pTColumn; STSchema *pTSchema; STSRow2 row; }; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 3f90fa36e6..c07d01b7d9 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -50,12 +50,101 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema * return -1; } - // (*ppTSchema) + (*ppTSchema)->numOfCols = ncols; + (*ppTSchema)->version = sver; + (*ppTSchema)->flen = 0; + (*ppTSchema)->vlen = 0; + (*ppTSchema)->tlen = 0; + + for (int32_t iCol = 0; iCol < ncols; iCol++) { + SSchema *pColumn = &pSchema[iCol]; + STColumn *pTColumn = &((*ppTSchema)->columns[iCol]); + + pTColumn->colId = pColumn->colId; + pTColumn->type = pColumn->type; + pTColumn->flags = pColumn->flags; + pTColumn->bytes = pColumn->bytes; + pTColumn->offset = (*ppTSchema)->flen; + + // skip first column + if (iCol) { + (*ppTSchema)->flen += TYPE_BYTES[pColumn->type]; + if (IS_VAR_DATA_TYPE(pColumn->type)) { + (*ppTSchema)->vlen += (pColumn->bytes + 5); + } + } + } + return 0; } void tTSchemaDestroy(STSchema *pTSchema) { taosMemoryFree(pTSchema); } +int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) { + // TODO + return 0; +} + +int32_t tTSRowBuilderClear(STSRowBuilder *pBuilder) { + // TODO + return 0; +} + +int32_t tTSRowBuilderReset(STSRowBuilder *pBuilder) { + for (int32_t iCol = pBuilder->pTSchema->numOfCols - 1; iCol >= 0; iCol--) { + pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; + + pBuilder->pTColumn->flags &= (~COL_VAL_SET); + } + + return 0; +} + +int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData) { + if (pBuilder->pTColumn->colId < cid) { + // right search + } else if (pBuilder->pTColumn->colId > cid) { + // left search + } + + // check if val is set already + if (pBuilder->pTColumn->flags & COL_VAL_SET) { + return -1; + } + + // set value + if (cid == 0) { + ASSERT(pData && nData == sizeof(TSKEY)); + pBuilder->row.ts = *(TSKEY *)pData; + } else { + if (pData) { + // set val + } else { + // set NULL val + } + } + + pBuilder->pTColumn->flags |= COL_VAL_SET; + + // TODO + return 0; +} + +int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { + if ((pBuilder->pTSchema->columns[0].flags & COL_VAL_SET) == 0) { + return -1; + } + + // chose which type row to return + + if (true /* tuple row is chosen */) { + // set non-set values as None + } + + *ppRow = &pBuilder->row; + return 0; +} + #if 1 // ==================== static void dataColSetNEleNull(SDataCol *pCol, int nEle); int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { From fc84b5c95572c2b4ad28c5b879d033c9179b5ba1 Mon Sep 17 00:00:00 2001 From: Shengliang Date: Wed, 11 May 2022 11:37:13 +0800 Subject: [PATCH 08/94] fix: alter stb --- source/dnode/mnode/impl/src/mndStb.c | 13 +++++++++---- tests/script/tsim/stable/alter1.sim | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index ca0ae111a3..12e89277f4 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -318,6 +318,7 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { pOld->updateTime = pNew->updateTime; pOld->version = pNew->version; pOld->nextColId = pNew->nextColId; + pOld->ttl = pNew->ttl; pOld->numOfColumns = pNew->numOfColumns; pOld->numOfTags = pNew->numOfTags; memcpy(pOld->pColumns, pNew->pColumns, pOld->numOfColumns * sizeof(SSchema)); @@ -832,7 +833,7 @@ static int32_t mndProcessVCreateStbRsp(SNodeMsg *pRsp) { } static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) { - if (pAlter->commentLen != 0) return 0; + if (pAlter->commentLen != 0 || pAlter->ttl != 0) return 0; if (pAlter->numOfFields < 1 || pAlter->numOfFields != (int32_t)taosArrayGetSize(pAlter->pFields)) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; @@ -883,7 +884,8 @@ static int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) { return 0; } -static int32_t mndUpdateStbComment(const SStbObj *pOld, SStbObj *pNew, char *pComment, int32_t commentLen) { +static int32_t mndUpdateStbCommentAndTTL(const SStbObj *pOld, SStbObj *pNew, char *pComment, int32_t commentLen, + int32_t ttl) { if (commentLen > 0) { pNew->commentLen = commentLen; pNew->comment = taosMemoryCalloc(1, commentLen); @@ -893,6 +895,9 @@ static int32_t mndUpdateStbComment(const SStbObj *pOld, SStbObj *pNew, char *pCo } memcpy(pNew->comment, pComment, commentLen); } + if (ttl >= 0) { + pNew->ttl = ttl; + } if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; @@ -1232,7 +1237,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SNodeMsg *pReq, const SMAlterStbReq * code = mndAlterStbColumnBytes(pOld, &stbObj, pField0); break; case TSDB_ALTER_TABLE_UPDATE_OPTIONS: - code = mndUpdateStbComment(pOld, &stbObj, pAlter->comment, pAlter->commentLen); + code = mndUpdateStbCommentAndTTL(pOld, &stbObj, pAlter->comment, pAlter->commentLen, pAlter->ttl); break; default: terrno = TSDB_CODE_OPS_NOT_SUPPORT; @@ -1723,7 +1728,7 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pStb->updateTime, false); // number of tables - char *p = taosMemoryMalloc(pStb->commentLen + VARSTR_HEADER_SIZE); // check malloc failures + char *p = taosMemoryCalloc(1, pStb->commentLen + 1 + VARSTR_HEADER_SIZE); // check malloc failures if (p != NULL) { if (pStb->commentLen != 0) { STR_TO_VARSTR(p, pStb->comment); diff --git a/tests/script/tsim/stable/alter1.sim b/tests/script/tsim/stable/alter1.sim index 5cee10756c..1205f50f6e 100644 --- a/tests/script/tsim/stable/alter1.sim +++ b/tests/script/tsim/stable/alter1.sim @@ -159,6 +159,7 @@ sql alter table db.stb rename tag t1 tx print ========== alter common sql alter table db.stb comment 'abcde' ; +sql alter table db.stb ttl 10 ; sql show db.stables; if $data[0][6] != abcde then From 057c7747bccf81d4a6e803746f5c449c1a115091 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 11 May 2022 11:40:04 +0800 Subject: [PATCH 09/94] use enterprise edition to test --- Jenkinsfile2 | 70 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index d827c6640a..c1f4827e77 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -47,36 +47,60 @@ def pre_test(){ script { if (env.CHANGE_TARGET == 'master') { sh ''' + cd ${WK} + git checkout master cd ${WKC} git checkout master ''' } else if(env.CHANGE_TARGET == '2.0') { sh ''' + cd ${WK} + git checkout 2.0 cd ${WKC} git checkout 2.0 ''' } else if(env.CHANGE_TARGET == '3.0') { sh ''' + cd ${WK} + git checkout 3.0 cd ${WKC} git checkout 3.0 [ -d contrib/bdb ] && cd contrib/bdb && git clean -fxd && cd ../.. ''' } else { sh ''' + cd ${WK} + git checkout develop cd ${WKC} git checkout develop ''' } } + if (env.CHANGE_URL =~ /\/TDengine\//) { + sh ''' + cd ${WKC} + git pull >/dev/null + git fetch origin +refs/pull/${CHANGE_ID}/merge + git checkout -qf FETCH_HEAD + ''' + } else if (env.CHANGE_URL =~ /\/TDinternal\//) { + sh ''' + cd ${WK} + git pull >/dev/null + git fetch origin +refs/pull/${CHANGE_ID}/merge + git checkout -qf FETCH_HEAD + ''' + } else { + sh ''' + echo "unmatched reposiotry ${CHANGE_URL}" + ''' + } sh ''' cd ${WKC} - git pull >/dev/null - git fetch origin +refs/pull/${CHANGE_ID}/merge - git checkout -qf FETCH_HEAD git submodule update --init --recursive ''' sh ''' - cd ${WKC} + cd ${WK} export TZ=Asia/Harbin date rm -rf debug @@ -162,8 +186,8 @@ pipeline { options { skipDefaultCheckout() } environment{ WK = '/var/lib/jenkins/workspace/TDinternal' - WKC= '/var/lib/jenkins/workspace/TDengine' - WKPY= '/var/lib/jenkins/workspace/taos-connector-python' + WKC = '/var/lib/jenkins/workspace/TDinternal/community' + WKPY = '/var/lib/jenkins/workspace/taos-connector-python' } stages { stage('run test') { @@ -177,15 +201,31 @@ pipeline { steps { timeout(time: 45, unit: 'MINUTES'){ pre_test() - sh ''' - cd ${WKC}/debug - ctest -VV - ''' - sh ''' - export LD_LIBRARY_PATH=${WKC}/debug/build/lib - cd ${WKC}/tests/system-test - ./fulltest.sh - ''' + if (env.CHANGE_URL =~ /\/TDengine\//) { + sh ''' + cd ${WK}/debug + ctest -VV + ''' + sh ''' + export LD_LIBRARY_PATH=${WK}/debug/build/lib + cd ${WKC}/tests/system-test + ./fulltest.sh + ''' + } else if (env.CHANGE_URL =~ /\/TDinternal\//) { + sh ''' + cd ${WKC}/debug + ctest -VV + ''' + sh ''' + export LD_LIBRARY_PATH=${WKC}/debug/build/lib + cd ${WKC}/tests/system-test + ./fulltest.sh + ''' + } else { + sh ''' + echo "unmatched reposiotry ${CHANGE_URL}" + ''' + } sh ''' cd ${WKC}/tests ./test-all.sh b1fq From 66bcbbba47ea53b0c9c132cdec47a22c7c7f7c55 Mon Sep 17 00:00:00 2001 From: dapan Date: Wed, 11 May 2022 11:44:09 +0800 Subject: [PATCH 10/94] stmt auto create table --- include/common/taosdef.h | 2 +- include/common/tmsg.h | 6 +- include/libs/scheduler/scheduler.h | 3 +- source/client/inc/clientInt.h | 4 +- source/client/inc/clientStmt.h | 5 +- source/client/src/clientImpl.c | 14 +- source/client/src/clientSml.c | 2 +- source/client/src/clientStmt.c | 135 ++++++++++++++++-- source/common/src/tmsg.c | 27 +++- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 4 +- source/dnode/vnode/src/vnd/vnodeQuery.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 67 +++++---- source/libs/parser/src/parInsert.c | 4 +- source/libs/parser/src/parInsertData.c | 2 +- source/libs/scheduler/inc/schedulerInt.h | 2 + source/libs/scheduler/src/scheduler.c | 56 ++++++-- source/libs/scheduler/test/schedulerTests.cpp | 2 +- tests/script/api/batchprepare.c | 8 +- 18 files changed, 259 insertions(+), 86 deletions(-) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index e1f8832edf..5384082da3 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -89,7 +89,7 @@ extern char *qtypeStr[]; #define TSDB_PORT_HTTP 11 -#undef TD_DEBUG_PRINT_ROW +#define TD_DEBUG_PRINT_ROW #ifdef __cplusplus } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ff9c2a0739..0b803bb572 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -254,10 +254,7 @@ int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq* pReq, STSchema* pSchema); typedef struct { int8_t hashMeta; int64_t uid; - union { - char* ename; // used for encode - const char* dname; // used for decode - }; + char* tblFName; int32_t numOfRows; int32_t affectedRows; } SSubmitBlkRsp; @@ -274,6 +271,7 @@ typedef struct { int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); +void tFreeSSubmitRsp(SSubmitRsp *pRsp); #define SCHEMA_SMA_ON 0x1 #define SCHEMA_IDX_ON 0x2 diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 460749243c..b3f35025d1 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -56,6 +56,7 @@ typedef struct SQueryResult { uint64_t numOfRows; int32_t msgSize; char *msg; + void *res; } SQueryResult; typedef struct STaskInfo { @@ -71,7 +72,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg); * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, SQueryResult *pRes); +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, bool needRes, SQueryResult *pRes); /** * Process the query job, generated according to the query physical plan. diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index b021651c16..ab11cfa6fd 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -307,9 +307,9 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v // --- mq void hbMgrInitMqHbRspHandle(); -SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery); +SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery, void** res); int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList); -int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList); +int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList, void** res); int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest); #ifdef __cplusplus diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 153cb84986..e5efafc214 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -55,6 +55,7 @@ typedef struct SStmtQueryResInfo { typedef struct SStmtBindInfo { bool needParse; + bool inExecCache; uint64_t tbUid; uint64_t tbSuid; int32_t sBindRowNum; @@ -64,6 +65,7 @@ typedef struct SStmtBindInfo { void* boundTags; char tbName[TSDB_TABLE_FNAME_LEN];; char tbFName[TSDB_TABLE_FNAME_LEN]; + char stbFName[TSDB_TABLE_FNAME_LEN]; SName sname; } SStmtBindInfo; @@ -72,12 +74,12 @@ typedef struct SStmtExecInfo { SRequestObj* pRequest; SHashObj* pVgHash; SHashObj* pBlockHash; + bool autoCreateTbl; } SStmtExecInfo; typedef struct SStmtSQLInfo { STMT_TYPE type; STMT_STATUS status; - bool autoCreateTbl; uint64_t runTimes; SHashObj* pTableCache; //SHash SQuery* pQuery; @@ -86,6 +88,7 @@ typedef struct SStmtSQLInfo { SArray* nodeList; SQueryPlan* pQueryPlan; SStmtQueryResInfo queryRes; + bool autoCreateTbl; } SStmtSQLInfo; typedef struct STscStmt { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 97c7d2bad1..d2e95d5eab 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -286,12 +286,12 @@ void setResPrecision(SReqResultInfo* pResInfo, int32_t precision) { pResInfo->precision = precision; } -int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) { +int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList, void** pRes) { void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, - pRequest->metric.start, &res); + pRequest->metric.start, NULL != pRes, &res); if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.queryJob != 0) { schedulerFreeJob(pRequest->body.queryJob); @@ -310,6 +310,10 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList } } + if (pRes) { + *pRes = res.res; + } + pRequest->code = res.code; terrno = res.code; return pRequest->code; @@ -320,7 +324,7 @@ int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList) return getPlan(pRequest, pQuery, &pRequest->body.pDag, *pNodeList); } -SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery) { +SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery, void** res) { if (TSDB_CODE_SUCCESS == code) { switch (pQuery->execMode) { case QUERY_EXEC_MODE_LOCAL: @@ -333,7 +337,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList); if (TSDB_CODE_SUCCESS == code) { - code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList); + code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList, res); } taosArrayDestroy(pNodeList); break; @@ -373,7 +377,7 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) { return pRequest; } - return launchQueryImpl(pRequest, pQuery, code, false); + return launchQueryImpl(pRequest, pQuery, code, false, NULL); } int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 59a7cbee1a..3656edaff3 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1668,7 +1668,7 @@ static int32_t smlInsertData(SSmlHandle* info) { smlBuildOutput(info->exec, info->pVgHash); info->cost.insertRpcTime = taosGetTimestampUs(); - launchQueryImpl(info->pRequest, info->pQuery, TSDB_CODE_SUCCESS, true); + launchQueryImpl(info->pRequest, info->pQuery, TSDB_CODE_SUCCESS, true, NULL); info->affectedRows = taos_affected_rows(info->pRequest); return info->pRequest->code; diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 61a850802a..d8262bc9a2 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -136,11 +136,12 @@ int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, return TSDB_CODE_SUCCESS; } -int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash) { +int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash, bool autoCreateTbl) { STscStmt* pStmt = (STscStmt*)stmt; pStmt->exec.pVgHash = pVgHash; pStmt->exec.pBlockHash = pBlockHash; + pStmt->exec.autoCreateTbl = autoCreateTbl; return TSDB_CODE_SUCCESS; } @@ -149,10 +150,10 @@ int32_t stmtUpdateInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, char STscStmt* pStmt = (STscStmt*)stmt; STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbFName)); - STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash)); - - pStmt->sql.autoCreateTbl = autoCreateTbl; + STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash, autoCreateTbl)); + pStmt->sql.autoCreateTbl = autoCreateTbl; + return TSDB_CODE_SUCCESS; } @@ -192,8 +193,12 @@ int32_t stmtCacheBlock(STscStmt *pStmt) { return TSDB_CODE_OUT_OF_MEMORY; } - pStmt->bInfo.boundTags = NULL; - + if (pStmt->sql.autoCreateTbl) { + pStmt->bInfo.tagsCached = true; + } else { + pStmt->bInfo.boundTags = NULL; + } + return TSDB_CODE_SUCCESS; } @@ -235,6 +240,7 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) { pStmt->bInfo.tbSuid = 0; pStmt->bInfo.tbType = 0; pStmt->bInfo.needParse = true; + pStmt->bInfo.inExecCache = false; pStmt->bInfo.tbName[0] = 0; pStmt->bInfo.tbFName[0] = 0; @@ -259,7 +265,7 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) { char *key = taosHashGetKey(pIter, &keyLen); STableMeta* pMeta = qGetTableMetaInDataBlock(pBlocks); - if (keepTable && (pMeta->uid == pStmt->bInfo.tbUid)) { + if (keepTable && (strlen(pStmt->bInfo.tbFName) == keyLen) && strncmp(pStmt->bInfo.tbFName, key, keyLen) == 0) { STMT_ERR_RET(qResetStmtDataBlock(pBlocks, true)); pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); @@ -272,6 +278,8 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) { pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); } + pStmt->exec.autoCreateTbl = false; + if (keepTable) { return TSDB_CODE_SUCCESS; } @@ -315,11 +323,45 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.needParse = true; + pStmt->bInfo.inExecCache = false; + + STableDataBlocks *pBlockInExec = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + if (pBlockInExec) { + pStmt->bInfo.inExecCache = true; + + if (pStmt->sql.autoCreateTbl) { + return TSDB_CODE_SUCCESS; + } + } if (NULL == pStmt->sql.pTableCache || taosHashGetSize(pStmt->sql.pTableCache) <= 0) { + if (pStmt->bInfo.inExecCache) { + ASSERT(taosHashGetSize(pStmt->exec.pBlockHash) == 1); + pStmt->bInfo.needParse = false; + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_SUCCESS; } + if (pStmt->sql.autoCreateTbl) { + SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pStmt->bInfo.tbSuid, sizeof(pStmt->bInfo.tbSuid)); + if (pCache) { + pStmt->bInfo.needParse = false; + + STableDataBlocks* pNewBlock = NULL; + STMT_ERR_RET(qRebuildStmtDataBlock(&pNewBlock, pCache->pDataBlock, 0)); + + if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { + STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; + } + + STMT_RET(stmtCleanBindInfo(pStmt)); + } + if (NULL == pStmt->pCatalog) { STMT_ERR_RET(catalogGetHandle(pStmt->taos->pAppInfo->clusterId, &pStmt->pCatalog)); } @@ -347,7 +389,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } - if (taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName))) { + if (pStmt->bInfo.inExecCache) { SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &cacheUid, sizeof(cacheUid)); if (NULL == pCache) { tscError("table [%s, %" PRIx64 ", %" PRIx64 "] found in exec blockHash, but not in sql blockHash", pStmt->bInfo.tbFName, uid, cacheUid); @@ -484,11 +526,13 @@ int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS)); - if (!pStmt->bInfo.needParse) { // table already cached, no need create table - return TSDB_CODE_SUCCESS; + if (pStmt->bInfo.needParse) { + STMT_ERR_RET(stmtParseSql(pStmt)); } - STMT_ERR_RET(stmtParseSql(pStmt)); + if (pStmt->bInfo.inExecCache) { + return TSDB_CODE_SUCCESS; + } STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { @@ -612,17 +656,73 @@ int stmtAddBatch(TAOS_STMT *stmt) { return TSDB_CODE_SUCCESS; } +int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp *pRsp) { + if (pRsp->nBlocks <= 0) { + tscError("invalid submit resp block number %d", pRsp->nBlocks); + STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + } + + size_t keyLen = 0; + STableDataBlocks **pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); + while (pIter) { + STableDataBlocks *pBlock = *pIter; + char *key = taosHashGetKey(pIter, &keyLen); + + STableMeta *pMeta = qGetTableMetaInDataBlock(pBlock); + if (pMeta->uid != pStmt->bInfo.tbUid) { + tscError("table uid %" PRIx64 " mis-match with current table uid %" PRIx64, pMeta->uid, pStmt->bInfo.tbUid); + STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + } + + if (pMeta->uid) { + pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); + continue; + } + + SSubmitBlkRsp *blkRsp = NULL; + int32_t i = 0; + for (; i < pRsp->nBlocks; ++i) { + blkRsp = pRsp->pBlocks + i; + if (strlen(blkRsp->tblFName) != keyLen) { + continue; + } + + if (strncmp(blkRsp->tblFName, key, keyLen)) { + continue; + } + + break; + } + + if (i < pRsp->nBlocks) { + tscDebug("auto created table %s uid updated from %" PRIx64 " to %" PRIx64, blkRsp->tblFName, pMeta->uid, blkRsp->uid); + + pMeta->uid = blkRsp->uid; + pStmt->bInfo.tbUid = blkRsp->uid; + } else { + tscError("table %s not found in submit rsp", pStmt->bInfo.tbFName); + STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + } + + pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); + } + + return TSDB_CODE_SUCCESS; +} + int stmtExec(TAOS_STMT *stmt) { STscStmt* pStmt = (STscStmt*)stmt; int32_t code = 0; + SSubmitRsp *pRsp = NULL; + bool autoCreateTbl = pStmt->exec.autoCreateTbl; STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); if (STMT_TYPE_QUERY == pStmt->sql.type) { - scheduleQuery(pStmt->exec.pRequest, pStmt->sql.pQueryPlan, pStmt->sql.nodeList); + scheduleQuery(pStmt->exec.pRequest, pStmt->sql.pQueryPlan, pStmt->sql.nodeList, NULL); } else { STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash)); - launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true); + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true, (autoCreateTbl ? (void**)&pRsp : NULL)); } if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) { @@ -643,6 +743,15 @@ int stmtExec(TAOS_STMT *stmt) { _return: stmtCleanExecInfo(pStmt, (code ? false : true), false); + + if (TSDB_CODE_SUCCESS == code && autoCreateTbl) { + if (NULL == pRsp) { + tscError("no submit resp got for auto create table"); + STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + } + + STMT_ERR_RET(stmtUpdateTableUid(pStmt, pRsp)); + } ++pStmt->sql.runTimes; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 8778a4cf6f..c68ce3c08b 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4022,7 +4022,7 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl if (tEncodeI8(pEncoder, pBlock->hashMeta) < 0) return -1; if (pBlock->hashMeta) { if (tEncodeI64(pEncoder, pBlock->uid) < 0) return -1; - if (tEncodeCStr(pEncoder, pBlock->ename) < 0) return -1; + if (tEncodeCStr(pEncoder, pBlock->tblFName) < 0) return -1; } if (tEncodeI32v(pEncoder, pBlock->numOfRows) < 0) return -1; if (tEncodeI32v(pEncoder, pBlock->affectedRows) < 0) return -1; @@ -4037,7 +4037,9 @@ static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { if (tDecodeI8(pDecoder, &pBlock->hashMeta) < 0) return -1; if (pBlock->hashMeta) { if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; - if (tDecodeCStr(pDecoder, &pBlock->dname) < 0) return -1; + pBlock->tblFName= taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); + if (NULL == pBlock->tblFName) return -1; + if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; } if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1; if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1; @@ -4068,12 +4070,29 @@ int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) { if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1; if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1; if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1; - pRsp->pBlocks = tDecoderMalloc(pDecoder, sizeof(*pRsp->pBlocks) * pRsp->nBlocks); + pRsp->pBlocks = taosMemoryCalloc(pRsp->nBlocks, sizeof(*pRsp->pBlocks)); if (pRsp->pBlocks == NULL) return -1; for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) { if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1; } - tEndDecode(pDecoder); + tEndDecode(pDecoder); + tDecoderClear(pDecoder); return 0; } + +void tFreeSSubmitRsp(SSubmitRsp *pRsp) { + if (NULL == pRsp) return; + + if (pRsp->pBlocks) { + for (int32_t i = 0; i < pRsp->nBlocks; ++i) { + SSubmitBlkRsp *sRsp = pRsp->pBlocks + i; + taosMemoryFree(sRsp->tblFName); + } + + taosMemoryFree(pRsp->pBlocks); + } + + taosMemoryFree(pRsp); +} + diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 95156c1e1c..466963f17f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -342,8 +342,8 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo if (pMemTable->keyMin > keyMin) pMemTable->keyMin = keyMin; if (pMemTable->keyMax < keyMax) pMemTable->keyMax = keyMax; - pRsp->numOfRows = pRsp->numOfRows; - pRsp->affectedRows = pRsp->affectedRows; + pRsp->numOfRows = pMsgIter->numOfRows; + pRsp->affectedRows = pMsgIter->numOfRows; return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 3e869650bf..8156e6c512 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -119,7 +119,7 @@ _exit: taosMemoryFree(metaRsp.pSchemas); metaReaderClear(&mer2); metaReaderClear(&mer1); - return code; + return TSDB_CODE_SUCCESS; } int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 678c9edbd4..10cdd9e85a 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -497,43 +497,51 @@ _exit: return 0; } +static int vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSubmitMsgIter *msgIter, const char *tags) { + SSubmitBlkIter blkIter = {0}; + STSchema *pSchema = NULL; + tb_uid_t suid = 0; + STSRow *row = NULL; + + tInitSubmitBlkIter(msgIter, pBlock, &blkIter); + if (blkIter.row == NULL) return 0; + if (!pSchema || (suid != msgIter->suid)) { + if (pSchema) { + taosMemoryFreeClear(pSchema); + } + pSchema = metaGetTbTSchema(pMeta, msgIter->suid, 0); // TODO: use the real schema + if (pSchema) { + suid = msgIter->suid; + } + } + if (!pSchema) { + printf("%s:%d no valid schema\n", tags, __LINE__); + return -1; + } + char __tags[128] = {0}; + snprintf(__tags, 128, "%s: uid %" PRIi64 " ", tags, msgIter->uid); + while ((row = tGetSubmitBlkNext(&blkIter))) { + tdSRowPrint(row, pSchema, __tags); + } + + taosMemoryFreeClear(pSchema); + + return TSDB_CODE_SUCCESS; +} + static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char *tags) { ASSERT(pMsg != NULL); SSubmitMsgIter msgIter = {0}; SMeta *pMeta = pVnode->pMeta; SSubmitBlk *pBlock = NULL; - SSubmitBlkIter blkIter = {0}; - STSRow *row = NULL; - STSchema *pSchema = NULL; - tb_uid_t suid = 0; if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; while (true) { if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; if (pBlock == NULL) break; - tInitSubmitBlkIter(&msgIter, pBlock, &blkIter); - if (blkIter.row == NULL) continue; - if (!pSchema || (suid != msgIter.suid)) { - if (pSchema) { - taosMemoryFreeClear(pSchema); - } - pSchema = metaGetTbTSchema(pMeta, msgIter.suid, 0); // TODO: use the real schema - if (pSchema) { - suid = msgIter.suid; - } - } - if (!pSchema) { - printf("%s:%d no valid schema\n", tags, __LINE__); - continue; - } - char __tags[128] = {0}; - snprintf(__tags, 128, "%s: uid %" PRIi64 " ", tags, msgIter.uid); - while ((row = tGetSubmitBlkNext(&blkIter))) { - tdSRowPrint(row, pSchema, __tags); - } - } - taosMemoryFreeClear(pSchema); + vnodeDebugPrintSingleSubmitMsg(pMeta, pBlock, &msgIter, tags); + } return 0; } @@ -589,8 +597,8 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in } submitBlkRsp.uid = createTbReq.uid; - submitBlkRsp.ename = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2); - sprintf(submitBlkRsp.ename, "%s.%s", pVnode->config.dbname, createTbReq.name); + submitBlkRsp.tblFName = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2); + sprintf(submitBlkRsp.tblFName, "%s.%s", pVnode->config.dbname, createTbReq.name); msgIter.uid = createTbReq.uid; if (createTbReq.type == TSDB_CHILD_TABLE) { @@ -599,6 +607,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in msgIter.suid = 0; } + vnodeDebugPrintSingleSubmitMsg(pVnode->pMeta, pBlock, &msgIter, "real uid"); tDecoderClear(&decoder); } @@ -621,7 +630,7 @@ _exit: tEncoderClear(&encoder); for (int32_t i = 0; i < taosArrayGetSize(submitRsp.pArray); i++) { - taosMemoryFree(((SSubmitBlkRsp *)taosArrayGet(submitRsp.pArray, i))[0].ename); + taosMemoryFree(((SSubmitBlkRsp *)taosArrayGet(submitRsp.pArray, i))[0].tblFName); } taosArrayDestroy(submitRsp.pArray); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index f1febb47cd..ad0319cf69 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -839,6 +839,7 @@ static int32_t storeTableMeta(SInsertParseContext* pCxt, SHashObj* pHash, SName* pMeta->uid = 0; pMeta->vgId = vg.vgId; + pMeta->tableType = TSDB_CHILD_TABLE; STableMeta* pBackup = NULL; if (TSDB_CODE_SUCCESS != cloneTableMeta(pMeta, &pBackup)) { @@ -1094,8 +1095,6 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { sToken.z = tbName; sToken.n = strlen(tbName); - - autoCreateTbl = true; } else { return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); } @@ -1112,6 +1111,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { if (TK_USING == sToken.type) { CHECK_CODE(parseUsingClause(pCxt, &name, tbFName)); NEXT_TOKEN(pCxt->pSql, sToken); + autoCreateTbl = true; } else { CHECK_CODE(getTableMeta(pCxt, &name, tbFName)); } diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 8bf1fe012e..d164df6db7 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -445,7 +445,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p const int INSERT_HEAD_SIZE = sizeof(SSubmitReq); int code = 0; bool isRawPayload = IS_RAW_PAYLOAD(payloadType); - SHashObj* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + SHashObj* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); STableDataBlocks** p = taosHashIterate(pHashObj, NULL); diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 5906ee8970..a90fb7fc2e 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -159,6 +159,7 @@ typedef struct SSchTask { typedef struct SSchJobAttr { EExplainMode explainMode; + bool needRes; bool syncSchedule; bool queryJob; bool needFlowCtrl; @@ -190,6 +191,7 @@ typedef struct SSchJob { SSchTask *fetchTask; int32_t errCode; SArray *errList; // SArray + SRWLatch resLock; void *resData; //TODO free it or not int32_t resNumOfRows; const char *sql; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index a8b0f2e2b8..1ce074c49f 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -70,7 +70,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * } int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray *pNodeList, const char *sql, - int64_t startTs, bool syncSchedule) { + int64_t startTs, bool needRes, bool syncSchedule) { int32_t code = 0; int64_t refId = -1; SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); @@ -81,6 +81,7 @@ int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray pJob->attr.explainMode = pDag->explainInfo.mode; pJob->attr.syncSchedule = syncSchedule; + pJob->attr.needRes = needRes; pJob->transport = transport; pJob->sql = sql; @@ -1133,16 +1134,39 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch break; } case TDMT_VND_SUBMIT_RSP: { - if (msg) { - SSubmitRsp *rsp = (SSubmitRsp *)msg; - // SCH_ERR_JRET(rsp->code); - } - SCH_ERR_JRET(rspCode); - SSubmitRsp *rsp = (SSubmitRsp *)msg; - if (rsp) { - pJob->resNumOfRows += rsp->affectedRows; + if (msg) { + SDecoder coder = {0}; + SSubmitRsp *rsp = taosMemoryMalloc(sizeof(*rsp)); + tDecoderInit(&coder, msg, msgSize); + code = tDecodeSSubmitRsp(&coder, rsp); + if (code) { + SCH_TASK_ELOG("decode submitRsp failed, code:%d", code); + tFreeSSubmitRsp(rsp); + SCH_ERR_JRET(code); + } + + atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows); + SCH_TASK_DLOG("submit succeed, affectedRows:%d", rsp->affectedRows); + + if (pJob->attr.needRes) { + SCH_LOCK(SCH_WRITE, &pJob->resLock); + if (pJob->resData) { + SSubmitRsp *sum = pJob->resData; + sum->affectedRows += rsp->affectedRows; + sum->nBlocks += rsp->nBlocks; + sum->pBlocks = taosMemoryRealloc(sum->pBlocks, sum->nBlocks * sizeof(*sum->pBlocks)); + memcpy(sum->pBlocks + sum->nBlocks - rsp->nBlocks, rsp->pBlocks, rsp->nBlocks * sizeof(*sum->pBlocks)); + taosMemoryFree(rsp->pBlocks); + taosMemoryFree(rsp); + } else { + pJob->resData = rsp; + } + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + } else { + tFreeSSubmitRsp(rsp); + } } SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); @@ -2350,7 +2374,7 @@ void schFreeJobImpl(void *job) { } static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, - int64_t startTs, bool syncSchedule) { + int64_t startTs, bool needRes, bool syncSchedule) { qDebug("QID:0x%" PRIx64 " job started", pDag->queryId); if (pNodeList == NULL || taosArrayGetSize(pNodeList) <= 0) { @@ -2359,7 +2383,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD int32_t code = 0; SSchJob *pJob = NULL; - SCH_ERR_JRET(schInitJob(&pJob, pDag, transport, pNodeList, sql, startTs, syncSchedule)); + SCH_ERR_JRET(schInitJob(&pJob, pDag, transport, pNodeList, sql, startTs, needRes, syncSchedule)); SCH_ERR_JRET(schLaunchJob(pJob)); @@ -2473,7 +2497,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { } int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, - int64_t startTs, SQueryResult *pRes) { + int64_t startTs, bool needRes, SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -2481,13 +2505,17 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) { SCH_ERR_RET(schExecStaticExplain(transport, nodeList, pDag, pJob, sql, true)); } else { - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, startTs, true)); + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, startTs, needRes, true)); } SSchJob *job = schAcquireJob(*pJob); pRes->code = atomic_load_32(&job->errCode); pRes->numOfRows = job->resNumOfRows; + if (needRes) { + pRes->res = job->resData; + job->resData = NULL; + } schReleaseJob(*pJob); @@ -2502,7 +2530,7 @@ int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryPlan *pD if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) { SCH_ERR_RET(schExecStaticExplain(transport, pNodeList, pDag, pJob, sql, false)); } else { - SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, 0, false)); + SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, 0, false, false)); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index fc0e05aaf1..09ecd9fffd 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -985,7 +985,7 @@ TEST(insertTest, normalCase) { taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId); SQueryResult res = {0}; - code = schedulerExecJob(mockPointer, qnodeList, &dag, &insertJobRefId, "insert into tb values(now,1)", 0, &res); + code = schedulerExecJob(mockPointer, qnodeList, &dag, &insertJobRefId, "insert into tb values(now,1)", 0, false, &res); ASSERT_EQ(code, 0); ASSERT_EQ(res.numOfRows, 20); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 5cccafa1b4..9016dca21e 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -213,7 +213,7 @@ CaseCtrl gCaseCtrl = { // default #if 1 -CaseCtrl gCaseCtrl = { // default +CaseCtrl gCaseCtrl = { .bindNullNum = 0, .printCreateTblSql = true, .printQuerySql = true, @@ -233,9 +233,9 @@ CaseCtrl gCaseCtrl = { // default .printRes = true, .runTimes = 0, .caseIdx = -1, - .caseNum = 1, - .caseRunIdx = 11, - .caseRunNum = 1, + .caseNum = 15, + .caseRunIdx = 8, + .caseRunNum = 15, }; #endif From 705bf811a7df2e7506bc8d31bef80b10f8404737 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 11 May 2022 11:49:27 +0800 Subject: [PATCH 11/94] [test: add test case for topic] --- tests/script/jenkins/basic.txt | 1 + tests/script/tsim/tmq/topic.sim | 93 +++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 tests/script/tsim/tmq/topic.sim diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 9f197b16c8..6297c7e6c2 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -75,6 +75,7 @@ ./test.sh -f tsim/tmq/basic3Of2Cons.sim ./test.sh -f tsim/tmq/basic4Of2Cons.sim ./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim +./test.sh -f tsim/tmq/topic.sim # --- stable ./test.sh -f tsim/stable/disk.sim diff --git a/tests/script/tsim/tmq/topic.sim b/tests/script/tsim/tmq/topic.sim new file mode 100644 index 0000000000..f1dbf98bb0 --- /dev/null +++ b/tests/script/tsim/tmq/topic.sim @@ -0,0 +1,93 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +sql connect +print == create database $dbName vgroups $vgroups +sql create database $dbName vgroups $vgroups + +#wait database ready +$loop_cnt = 0 +check_db_ready: +if $loop_cnt == 10 then + print ====> database not ready! + return -1 +endi +sql show databases +print ==> rows: $rows +print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] +print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] +if $data(db)[19] != nostrict then + sleep 100 + $loop_cnt = $loop_cnt + 1 + goto check_db_ready +endi + +sql use $dbName + + +print == create super table +sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) +sql show stables +if $rows != 1 then + return -1 +endi + +print == create child table, normal table and insert data +$i = 0 +while $i < $ctbNum + $ctb = $ctbPrefix . $i + $ntb = $ntbPrefix . $i + sql create table $ctb using $stbPrefix tags( $i ) + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) + $i = $i + 1 +endw + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + + +print == show topics +sql show topics +if $rows != 9 then + return -1 +endi + +print == drop topic +sql drop topic topic_stb_column +sql drop topic topic_ctb_column +sql drop topic topic_ntb_column + +print == show topics +sql show topics +if $rows != 6 then + return -1 +endi + + From a36a7d68a7554cbec07e820033f91050283904a8 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 11 May 2022 12:00:29 +0800 Subject: [PATCH 12/94] correct syntax error --- Jenkinsfile2 | 50 ++++++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index c1f4827e77..4f64c4f817 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -201,30 +201,32 @@ pipeline { steps { timeout(time: 45, unit: 'MINUTES'){ pre_test() - if (env.CHANGE_URL =~ /\/TDengine\//) { - sh ''' - cd ${WK}/debug - ctest -VV - ''' - sh ''' - export LD_LIBRARY_PATH=${WK}/debug/build/lib - cd ${WKC}/tests/system-test - ./fulltest.sh - ''' - } else if (env.CHANGE_URL =~ /\/TDinternal\//) { - sh ''' - cd ${WKC}/debug - ctest -VV - ''' - sh ''' - export LD_LIBRARY_PATH=${WKC}/debug/build/lib - cd ${WKC}/tests/system-test - ./fulltest.sh - ''' - } else { - sh ''' - echo "unmatched reposiotry ${CHANGE_URL}" - ''' + script { + if (env.CHANGE_URL =~ /\/TDengine\//) { + sh ''' + cd ${WK}/debug + ctest -VV + ''' + sh ''' + export LD_LIBRARY_PATH=${WK}/debug/build/lib + cd ${WKC}/tests/system-test + ./fulltest.sh + ''' + } else if (env.CHANGE_URL =~ /\/TDinternal\//) { + sh ''' + cd ${WKC}/debug + ctest -VV + ''' + sh ''' + export LD_LIBRARY_PATH=${WKC}/debug/build/lib + cd ${WKC}/tests/system-test + ./fulltest.sh + ''' + } else { + sh ''' + echo "unmatched reposiotry ${CHANGE_URL}" + ''' + } } sh ''' cd ${WKC}/tests From cdd7cafe42dcb736c8c5807c989b0940bfedd84b Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 11 May 2022 12:01:37 +0800 Subject: [PATCH 13/94] fix(os): Ubuntu timezone error. --- source/os/src/osTimezone.c | 105 ++++++++++++++++++++++++++++++++++++- 1 file changed, 104 insertions(+), 1 deletion(-) diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 575d5bc187..06ec7e9464 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -125,7 +125,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { strcpy(outTimezoneStr, tz); } -#else +#elif defined(_TD_DARWIN_64) char buf[4096] = {0}; char *tz = NULL; { @@ -170,5 +170,108 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], -timezone / 3600); +#else + + if (taosCheckExistFile("/etc/timezone")) { + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = taosGetTimestampSec(); + struct tm tm1; + taosLocalTime(&tx1, &tm1); + /* load time zone string from /etc/timezone */ + // FILE *f = fopen("/etc/timezone", "r"); + errno = 0; + TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ); + char buf[68] = {0}; + if (pFile != NULL) { + int len = taosReadFile(pFile, buf, 64); + if (len < 64 && taosGetErrorFile(pFile)) { + taosCloseFile(&pFile); + printf("read /etc/timezone error, reason:%s", strerror(errno)); + return; + } + + taosCloseFile(&pFile); + + buf[sizeof(buf) - 1] = 0; + char *lineEnd = strstr(buf, "\n"); + if (lineEnd != NULL) { + *lineEnd = 0; + } + + // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables + if (strlen(buf) > 0) { + setenv("TZ", buf, 1); + } + } + // get and set default timezone + tzset(); + /* + * get CURRENT time zone. + * system current time zone is affected by daylight saving time(DST) + * + * e.g., the local time zone of London in DST is GMT+01:00, + * otherwise is GMT+00:00 + */ + int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; + *tsTimezone = tz; + tz += daylight; + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + } else { + char buf[4096] = {0}; + char *tz = NULL; + { + int n = readlink("/etc/localtime", buf, sizeof(buf)); + if (n < 0) { + printf("read /etc/localtime error, reason:%s", strerror(errno)); + return; + } + buf[n] = '\0'; + for (int i = n - 1; i >= 0; --i) { + if (buf[i] == '/') { + if (tz) { + tz = buf + i + 1; + break; + } + tz = buf + i + 1; + } + } + if (!tz || 0 == strchr(tz, '/')) { + printf("parsing /etc/localtime failed"); + return; + } + + setenv("TZ", tz, 1); + tzset(); + } + + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = taosGetTimestampSec(); + struct tm tm1; + taosLocalTime(&tx1, &tm1); + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], + -timezone / 3600); + } #endif } From 675bf2aabdf56dc2c54ad9633853b6db44196431 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 11 May 2022 12:05:56 +0800 Subject: [PATCH 14/94] remove bdb related --- Jenkinsfile2 | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 4f64c4f817..79b27dda14 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -65,7 +65,6 @@ def pre_test(){ git checkout 3.0 cd ${WKC} git checkout 3.0 - [ -d contrib/bdb ] && cd contrib/bdb && git clean -fxd && cd ../.. ''' } else { sh ''' From c3cf7f52bdf4befe196e4716b0ca65743b1e6fd8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 May 2022 04:18:23 +0000 Subject: [PATCH 15/94] refact data format --- include/common/tdataformat.h | 7 +++++ source/common/src/tdataformat.c | 46 +++++++++++++++++++++++++++++---- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 0aa09d0d06..e270e09471 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -31,7 +31,9 @@ typedef struct STColumn STColumn; typedef struct STSchema STSchema; typedef struct STSRow2 STSRow2; typedef struct STSRowBuilder STSRowBuilder; +typedef struct SKVIdx SKVIdx; +#define TD_TP_ROW 0x0U #define TD_KV_ROW 0x1U // STSchema @@ -83,6 +85,11 @@ struct STSRow2 { struct STSRowBuilder { STColumn *pTColumn; STSchema *pTSchema; + int32_t nCols; + int32_t kvVLen; + uint8_t *pKV; + int32_t tpVLen; + uint8_t *pTuple; STSRow2 row; }; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index c07d01b7d9..c39ad3821f 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -19,6 +19,11 @@ #include "tdatablock.h" #include "tlog.h" +struct SKVIdx { + int32_t cid; + int32_t offset; +}; + int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) { if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1; if (tEncodeU32v(pEncoder, pRow->flags) < 0) return -1; @@ -97,6 +102,11 @@ int32_t tTSRowBuilderReset(STSRowBuilder *pBuilder) { pBuilder->pTColumn->flags &= (~COL_VAL_SET); } + pBuilder->nCols = 0; + pBuilder->kvVLen = 0; + pBuilder->tpVLen = 0; + pBuilder->row.flags = 0; + return 0; } @@ -125,8 +135,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD } pBuilder->pTColumn->flags |= COL_VAL_SET; - - // TODO + pBuilder->nCols++; return 0; } @@ -135,10 +144,37 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { return -1; } - // chose which type row to return + if (pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen < pBuilder->pTSchema->flen + pBuilder->tpVLen) { + // encode as TD_KV_ROW + pBuilder->row.flags |= TD_KV_ROW; + pBuilder->row.ncols = pBuilder->nCols; + pBuilder->row.nData = pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen; + pBuilder->row.pData = pBuilder->pKV; - if (true /* tuple row is chosen */) { - // set non-set values as None + if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) { + memmove(pBuilder->pKV + sizeof(SKVIdx) * pBuilder->nCols, + pBuilder->pKV + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols, pBuilder->kvVLen); + } + } else { + // encode as TD_TUPLE_ROW + pBuilder->row.flags &= (~TD_KV_ROW); + pBuilder->row.sver = pBuilder->pTSchema->version; + pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen; + pBuilder->row.pData = pBuilder->pTuple; + + if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) { + // set non-set cols as None + for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) { + pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; + if (pBuilder->pTColumn->flags & COL_VAL_SET) continue; + + { + // set None (todo) + } + + pBuilder->pTColumn->flags |= COL_VAL_SET; + } + } } *ppRow = &pBuilder->row; From 666b00170bf936715d8ca1c4c4f92f77793764e2 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 11 May 2022 12:26:46 +0800 Subject: [PATCH 16/94] print git log --- Jenkinsfile2 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 79b27dda14..d90bc3061b 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -81,6 +81,10 @@ def pre_test(){ git pull >/dev/null git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD + git log|head -n20 + cd ${WK} + git pull >/dev/null + git log|head -n20 ''' } else if (env.CHANGE_URL =~ /\/TDinternal\//) { sh ''' @@ -88,6 +92,10 @@ def pre_test(){ git pull >/dev/null git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD + git log|head -n20 + cd ${WKC} + git pull >/dev/null + git log|head -n20 ''' } else { sh ''' From abc4973707c08efcec6c3adf4b51f98da84718cc Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 11 May 2022 12:39:15 +0800 Subject: [PATCH 17/94] test: add test case for select topic info --- .../script/tsim/tmq/basic2Of2ConsOverlap.sim | 684 +++++++++--------- 1 file changed, 347 insertions(+), 337 deletions(-) diff --git a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim index 878e3d9031..d5c800b0e9 100644 --- a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim +++ b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim @@ -1,337 +1,347 @@ -#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 -#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic2Of2ConsOverlap.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics -#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics -#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics - -# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN -# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; -# -# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). -# - -run tsim/tmq/prepareBasicEnv-1vgrp.sim - -#---- global parameters start ----# -$dbName = db -$vgroups = 1 -$stbPrefix = stb -$ctbPrefix = ctb -$ntbPrefix = ntb -$stbNum = 1 -$ctbNum = 10 -$ntbNum = 10 -$rowsPerCtb = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 -#---- global parameters end ----# - -$pullDelay = 5 -$ifcheckdata = 1 -$showMsg = 1 -$showRow = 0 - -sql connect -sql use $dbName - -print == create topics from super table -sql create topic topic_stb_column as select ts, c3 from stb -sql create topic topic_stb_all as select ts, c1, c2, c3 from stb -sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb - -print == create topics from child table -sql create topic topic_ctb_column as select ts, c3 from ctb0 -sql create topic topic_ctb_all as select * from ctb0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 - -print == create topics from normal table -sql create topic topic_ntb_column as select ts, c3 from ntb0 -sql create topic topic_ntb_all as select * from ntb0 -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 - -#sql show topics -#if $rows != 9 then -# return -1 -#endi - -$keyList = ' . group.id:cgrp1 -$keyList = $keyList . ' - -$topicNum = 2 - -#=============================== start consume =============================# - - -print ================ test consume from stb -print == overlap toipcs: topic_stb_column + topic_stb_all, topic_stb_function + topic_stb_all -$topicList = ' . topic_stb_column -$topicList = $topicList . , -$topicList = $topicList . topic_stb_all -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfOneTopic = $ctbNum * $rowsPerCtb -$totalMsgOfStb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - - -$topicList = ' . topic_stb_all -$topicList = $topicList . , -$topicList = $topicList . topic_stb_function -$topicList = $topicList . ' -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from stb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start - -print == check consume result -wait_consumer_end_from_stb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_stb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# $data[0][2]/$data[1][2] should be between $totalMsgOfOneTopic and $totalMsgOfStb. - -if $data[0][2] < $totalMsgOfOneTopic then - return -1 -endi -if $data[0][2] > $totalMsgOfStb then - return -1 -endi -if $data[1][2] < $totalMsgOfOneTopic then - return -1 -endi -if $data[1][2] > $totalMsgOfStb then - return -1 -endi - -$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb -$sumOfMsgCnt = $data[0][2] + $data[1][2] -if $sumOfMsgCnt != $totalMsgCons then - return -1 -endi - -# $data[0][3]/$data[1][3] should be between $totalMsgOfOneTopic and $totalMsgOfStb. -if $data[0][3] < $totalMsgOfOneTopic then - return -1 -endi -if $data[0][3] > $totalMsgOfStb then - return -1 -endi -if $data[1][3] < $totalMsgOfOneTopic then - return -1 -endi -if $data[1][3] > $totalMsgOfStb then - return -1 -endi - -$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb -$sumOfRows = $data[0][3] + $data[1][3] -if $sumOfRows != $totalMsgCons then - return -1 -endi - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb1 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table for ctb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ctb -print == overlap toipcs: topic_ctb_column + topic_ctb_all, topic_ctb_function + topic_ctb_all -$topicList = ' . topic_ctb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_all -$topicList = $topicList . ' -$consumerId = 0 - -$totalMsgOfOneTopic = $rowsPerCtb -$totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -$topicList = ' . topic_ctb_function -$topicList = $topicList . , -$topicList = $topicList . topic_ctb_all -$topicList = $topicList . ' -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ctb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result -wait_consumer_end_from_ctb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ctb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfCtb -# or $data[0][2] $totalMsgOfCtb and $data[1][2] == $totalMsgOfOneTopic -if $data[0][2] == $totalMsgOfOneTopic then - if $data[1][2] == $totalMsgOfCtb then - goto check_ok_0 - endi -elif $data[1][2] == $totalMsgOfOneTopic then - if $data[0][2] == $totalMsgOfCtb then - goto check_ok_0 - endi -endi -return -1 -check_ok_0: - -if $data[0][3] == $totalMsgOfOneTopic then - if $data[1][3] == $totalMsgOfCtb then - goto check_ok_1 - endi -elif $data[1][3] == $totalMsgOfOneTopic then - if $data[0][3] == $totalMsgOfCtb then - goto check_ok_1 - endi -endi -return -1 -check_ok_1: - -####################################################################################### -# clear consume info and consume result -#run tsim/tmq/clearConsume.sim -# because drop table function no stable, so by create new db for consume info and result. Modify it later -$cdbName = cdb2 -sql create database $cdbName vgroups 1 -sleep 500 -sql use $cdbName - -print == create consume info table and consume result table for ntb -sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) -sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) - -sql show tables -if $rows != 2 then - return -1 -endi -####################################################################################### - - -print ================ test consume from ntb -print == overlap toipcs: topic_ntb_column + topic_ntb_all, topic_ntb_function + topic_ntb_all -$topicList = ' . topic_ntb_column -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_all -$topicList = $topicList . ' - -$consumerId = 0 -$totalMsgOfOneTopic = $rowsPerCtb -$totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - - -$topicList = ' . topic_ntb_function -$topicList = $topicList . , -$topicList = $topicList . topic_ntb_all -$topicList = $topicList . ' -$consumerId = 1 -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) - -print == start consumer to pull msgs from ntb -print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start - -print == check consume result from ntb -wait_consumer_end_from_ntb: -sql select * from consumeresult -print ==> rows: $rows -print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -if $rows != 2 then - sleep 1000 - goto wait_consumer_end_from_ntb -endi -if $data[0][1] == 0 then - if $data[1][1] != 1 then - return -1 - endi -endi -if $data[0][1] == 1 then - if $data[1][1] != 0 then - return -1 - endi -endi - -# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfNtb -# or $data[0][2] $totalMsgOfNtb and $data[1][2] == $totalMsgOfOneTopic -if $data[0][2] == $totalMsgOfOneTopic then - if $data[1][2] == $totalMsgOfNtb then - goto check_ok_2 - endi -elif $data[1][2] == $totalMsgOfOneTopic then - if $data[0][2] == $totalMsgOfNtb then - goto check_ok_2 - endi -endi -return -1 -check_ok_2: - -if $data[0][3] == $totalMsgOfOneTopic then - if $data[1][3] == $totalMsgOfNtb then - goto check_ok_3 - endi -elif $data[1][3] == $totalMsgOfOneTopic then - if $data[0][3] == $totalMsgOfNtb then - goto check_ok_3 - endi -endi -return -1 -check_ok_3: - -#------ not need stop consumer, because it exit after pull msg overthan expect msg -#system tsim/tmq/consume.sh -s stop -x SIGINT - -system sh/exec.sh -n dnode1 -s stop -x SIGINT +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic2Of2ConsOverlap.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics +#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-1vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 5 +$ifcheckdata = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . ' + +$topicNum = 2 + +#=============================== start consume =============================# + + +print ================ test consume from stb +print == overlap toipcs: topic_stb_column + topic_stb_all, topic_stb_function + topic_stb_all +$topicList = ' . topic_stb_column +$topicList = $topicList . , +$topicList = $topicList . topic_stb_all +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfOneTopic = $ctbNum * $rowsPerCtb +$totalMsgOfStb = $totalMsgOfOneTopic * $topicNum +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + + +$topicList = ' . topic_stb_all +$topicList = $topicList . , +$topicList = $topicList . topic_stb_function +$topicList = $topicList . ' +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# $data[0][2]/$data[1][2] should be between $totalMsgOfOneTopic and $totalMsgOfStb. + +if $data[0][2] < $totalMsgOfOneTopic then + return -1 +endi +if $data[0][2] > $totalMsgOfStb then + return -1 +endi +if $data[1][2] < $totalMsgOfOneTopic then + return -1 +endi +if $data[1][2] > $totalMsgOfStb then + return -1 +endi + +$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb +$sumOfMsgCnt = $data[0][2] + $data[1][2] +if $sumOfMsgCnt != $totalMsgCons then + return -1 +endi + +# $data[0][3]/$data[1][3] should be between $totalMsgOfOneTopic and $totalMsgOfStb. +if $data[0][3] < $totalMsgOfOneTopic then + return -1 +endi +if $data[0][3] > $totalMsgOfStb then + return -1 +endi +if $data[1][3] < $totalMsgOfOneTopic then + return -1 +endi +if $data[1][3] > $totalMsgOfStb then + return -1 +endi + +$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb +$sumOfRows = $data[0][3] + $data[1][3] +if $sumOfRows != $totalMsgCons then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb1 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table for ctb +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ctb +print == overlap toipcs: topic_ctb_column + topic_ctb_all, topic_ctb_function + topic_ctb_all +$topicList = ' . topic_ctb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . ' +$consumerId = 0 + +$totalMsgOfOneTopic = $rowsPerCtb +$totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +$topicList = ' . topic_ctb_function +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . ' +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfCtb +# or $data[0][2] $totalMsgOfCtb and $data[1][2] == $totalMsgOfOneTopic +if $data[0][2] == $totalMsgOfOneTopic then + if $data[1][2] == $totalMsgOfCtb then + goto check_ok_0 + endi +elif $data[1][2] == $totalMsgOfOneTopic then + if $data[0][2] == $totalMsgOfCtb then + goto check_ok_0 + endi +endi +return -1 +check_ok_0: + +if $data[0][3] == $totalMsgOfOneTopic then + if $data[1][3] == $totalMsgOfCtb then + goto check_ok_1 + endi +elif $data[1][3] == $totalMsgOfOneTopic then + if $data[0][3] == $totalMsgOfCtb then + goto check_ok_1 + endi +endi +return -1 +check_ok_1: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb2 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table for ntb +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ntb +print == overlap toipcs: topic_ntb_column + topic_ntb_all, topic_ntb_function + topic_ntb_all +$topicList = ' . topic_ntb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfOneTopic = $rowsPerCtb +$totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + + +$topicList = ' . topic_ntb_function +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . ' +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfNtb +# or $data[0][2] $totalMsgOfNtb and $data[1][2] == $totalMsgOfOneTopic +if $data[0][2] == $totalMsgOfOneTopic then + if $data[1][2] == $totalMsgOfNtb then + goto check_ok_2 + endi +elif $data[1][2] == $totalMsgOfOneTopic then + if $data[0][2] == $totalMsgOfNtb then + goto check_ok_2 + endi +endi +return -1 +check_ok_2: + +if $data[0][3] == $totalMsgOfOneTopic then + if $data[1][3] == $totalMsgOfNtb then + goto check_ok_3 + endi +elif $data[1][3] == $totalMsgOfOneTopic then + if $data[0][3] == $totalMsgOfNtb then + goto check_ok_3 + endi +endi +return -1 +check_ok_3: + +sql select * from performance_schema.`consumers` +if $rows != 0 then + return -1 +endi + +#sql select * from performance_schema.`subscriptions` +#if $rows != 0 then +# return -1 +#endi + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 84d4bf6a0775fc2ababac0223d91ce90574a94fb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 May 2022 06:14:22 +0000 Subject: [PATCH 18/94] fix database options --- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index c5919e06b6..88c0328bcb 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -140,11 +140,17 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->vgId = pCreate->vgId; tstrncpy(pCfg->dbname, pCreate->db, sizeof(pCfg->dbname)); pCfg->dbId = pCreate->dbUid; + pCfg->szPage = pCreate->pageSize * 1024; + pCfg->szCache = pCreate->pages; + pCfg->szBuf = pCreate->buffer; pCfg->isWeak = true; + pCfg->tsdbCfg.precision = pCreate->precision; pCfg->tsdbCfg.days = 10; pCfg->tsdbCfg.keep0 = 3650; pCfg->tsdbCfg.keep1 = 3650; pCfg->tsdbCfg.keep2 = 3650; + pCfg->tsdbCfg.minRows = pCreate->minRows; + pCfg->tsdbCfg.maxRows = pCreate->maxRows; for (size_t i = 0; i < taosArrayGetSize(pCreate->pRetensions); ++i) { memcpy(&pCfg->tsdbCfg.retentions[i], taosArrayGet(pCreate->pRetensions, i), sizeof(SRetention)); } From f5cf45be098a886e99f42804979d151b1b9569d9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 May 2022 06:21:22 +0000 Subject: [PATCH 19/94] fix: database options --- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 88c0328bcb..6a0f4984b1 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -142,7 +142,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->dbId = pCreate->dbUid; pCfg->szPage = pCreate->pageSize * 1024; pCfg->szCache = pCreate->pages; - pCfg->szBuf = pCreate->buffer; + pCfg->szBuf = pCreate->buffer * 1024 * 1024; pCfg->isWeak = true; pCfg->tsdbCfg.precision = pCreate->precision; pCfg->tsdbCfg.days = 10; From f506604d653ccc832ab2dd1af915eb2a881c454e Mon Sep 17 00:00:00 2001 From: dapan Date: Wed, 11 May 2022 14:41:39 +0800 Subject: [PATCH 20/94] stmt auto create table --- include/libs/parser/parser.h | 2 +- source/client/src/clientStmt.c | 25 +++++++++++++++++++------ source/libs/parser/src/parInsertData.c | 5 ++++- tests/script/api/batchprepare.c | 13 +++++++------ 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 04f441e0e4..61e4bb3723 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -58,7 +58,7 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash int32_t qResetStmtDataBlock(void* block, bool keepBuf); int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); void qFreeStmtDataBlock(void* pDataBlock); -int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid); +int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId); void qDestroyStmtDataBlock(void* pBlock); STableMeta *qGetTableMetaInDataBlock(void* pDataBlock); diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index d8262bc9a2..3a2363f882 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -321,6 +321,18 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } +int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataBlocks *pDataBlock, STableDataBlocks **newBlock, uint64_t uid) { + SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); + SVgroupInfo vgInfo = {0}; + + STMT_ERR_RET(catalogGetTableHashVgroup(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &vgInfo)); + STMT_ERR_RET(taosHashPut(pStmt->exec.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo))); + + STMT_ERR_RET(qRebuildStmtDataBlock(newBlock, pDataBlock, uid, vgInfo.vgId)); + + return TSDB_CODE_SUCCESS; +} + int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.needParse = true; pStmt->bInfo.inExecCache = false; @@ -344,14 +356,18 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } + if (NULL == pStmt->pCatalog) { + STMT_ERR_RET(catalogGetHandle(pStmt->taos->pAppInfo->clusterId, &pStmt->pCatalog)); + } + if (pStmt->sql.autoCreateTbl) { SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pStmt->bInfo.tbSuid, sizeof(pStmt->bInfo.tbSuid)); if (pCache) { pStmt->bInfo.needParse = false; STableDataBlocks* pNewBlock = NULL; - STMT_ERR_RET(qRebuildStmtDataBlock(&pNewBlock, pCache->pDataBlock, 0)); - + STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataBlock, &pNewBlock, 0)); + if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -362,9 +378,6 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { STMT_RET(stmtCleanBindInfo(pStmt)); } - if (NULL == pStmt->pCatalog) { - STMT_ERR_RET(catalogGetHandle(pStmt->taos->pAppInfo->clusterId, &pStmt->pCatalog)); - } STableMeta *pTableMeta = NULL; SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); @@ -419,7 +432,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.tagsCached = true; STableDataBlocks* pNewBlock = NULL; - STMT_ERR_RET(qRebuildStmtDataBlock(&pNewBlock, pCache->pDataBlock, uid)); + STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataBlock, &pNewBlock, uid)); if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index d164df6db7..8deaad6091 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -620,7 +620,7 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) { return qResetStmtDataBlock(*pDst, false); } -int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid) { +int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId) { int32_t code = qCloneStmtDataBlock(pDst, pSrc); if (code) { return code; @@ -633,8 +633,11 @@ int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid) { return TSDB_CODE_OUT_OF_MEMORY; } + pBlock->vgId = vgId; + if (pBlock->pTableMeta) { pBlock->pTableMeta->uid = uid; + pBlock->pTableMeta->vgId = vgId; } memset(pBlock->pData, 0, sizeof(SSubmitBlk)); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 9016dca21e..4423ee3bc8 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -230,12 +230,12 @@ CaseCtrl gCaseCtrl = { .optrIdxListNum = 0, .optrIdxList = NULL, .checkParamNum = false, - .printRes = true, + .printRes = false, .runTimes = 0, .caseIdx = -1, - .caseNum = 15, - .caseRunIdx = 8, - .caseRunNum = 15, + .caseNum = -1, + .caseRunIdx = -1, + .caseRunNum = -1, }; #endif @@ -3465,16 +3465,17 @@ void* runCaseList(TAOS *taos) { } void runAll(TAOS *taos) { -/* strcpy(gCaseCtrl.caseCatalog, "Normal Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); runCaseList(taos); -*/ + +#if 0 strcpy(gCaseCtrl.caseCatalog, "Auto Create Table Test"); gCaseCtrl.autoCreateTbl = true; printf("%s Begin\n", gCaseCtrl.caseCatalog); runCaseList(taos); gCaseCtrl.autoCreateTbl = false; +#endif /* strcpy(gCaseCtrl.caseCatalog, "Null Test"); From 714dd4f497dbf99fde567b104249b8756ca15832 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 11 May 2022 14:42:20 +0800 Subject: [PATCH 21/94] enh(tmq): only read committed log --- include/common/tmsg.h | 355 +++++++++------------ include/libs/wal/wal.h | 1 + source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/src/mndSubscribe.c | 3 + source/dnode/mnode/impl/src/mndTopic.c | 4 + source/dnode/vnode/src/tq/tq.c | 2 +- source/libs/wal/src/walMeta.c | 2 + source/libs/wal/src/walRead.c | 4 + 8 files changed, 172 insertions(+), 200 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ff2e419c75..b50367af03 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -291,6 +291,109 @@ typedef struct { SSchema* pSchema; } SSchemaWrapper; +static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* pSchemaWrapper) { + SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper)); + if (pSW == NULL) return pSW; + pSW->nCols = pSchemaWrapper->nCols; + pSW->sver = pSchemaWrapper->sver; + pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); + if (pSW->pSchema == NULL) { + taosMemoryFree(pSW); + return NULL; + } + memcpy(pSW->pSchema, pSchemaWrapper->pSchema, pSW->nCols * sizeof(SSchema)); + return pSW; +} + +static FORCE_INLINE void tDeleteSSchemaWrapper(SSchemaWrapper* pSchemaWrapper) { + taosMemoryFree(pSchemaWrapper->pSchema); + taosMemoryFree(pSchemaWrapper); +} + +static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) { + int32_t tlen = 0; + tlen += taosEncodeFixedI8(buf, pSchema->type); + tlen += taosEncodeFixedI8(buf, pSchema->flags); + tlen += taosEncodeFixedI32(buf, pSchema->bytes); + tlen += taosEncodeFixedI16(buf, pSchema->colId); + tlen += taosEncodeString(buf, pSchema->name); + return tlen; +} + +static FORCE_INLINE void* taosDecodeSSchema(const void* buf, SSchema* pSchema) { + buf = taosDecodeFixedI8(buf, &pSchema->type); + buf = taosDecodeFixedI8(buf, &pSchema->flags); + buf = taosDecodeFixedI32(buf, &pSchema->bytes); + buf = taosDecodeFixedI16(buf, &pSchema->colId); + buf = taosDecodeStringTo(buf, pSchema->name); + return (void*)buf; +} + +static FORCE_INLINE int32_t tEncodeSSchema(SEncoder* pEncoder, const SSchema* pSchema) { + if (tEncodeI8(pEncoder, pSchema->type) < 0) return -1; + if (tEncodeI8(pEncoder, pSchema->flags) < 0) return -1; + if (tEncodeI32v(pEncoder, pSchema->bytes) < 0) return -1; + if (tEncodeI16v(pEncoder, pSchema->colId) < 0) return -1; + if (tEncodeCStr(pEncoder, pSchema->name) < 0) return -1; + return 0; +} + +static FORCE_INLINE int32_t tDecodeSSchema(SDecoder* pDecoder, SSchema* pSchema) { + if (tDecodeI8(pDecoder, &pSchema->type) < 0) return -1; + if (tDecodeI8(pDecoder, &pSchema->flags) < 0) return -1; + if (tDecodeI32v(pDecoder, &pSchema->bytes) < 0) return -1; + if (tDecodeI16v(pDecoder, &pSchema->colId) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pSchema->name) < 0) return -1; + return 0; +} + +static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) { + int32_t tlen = 0; + tlen += taosEncodeVariantI32(buf, pSW->nCols); + tlen += taosEncodeVariantI32(buf, pSW->sver); + for (int32_t i = 0; i < pSW->nCols; i++) { + tlen += taosEncodeSSchema(buf, &pSW->pSchema[i]); + } + return tlen; +} + +static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapper* pSW) { + buf = taosDecodeVariantI32(buf, &pSW->nCols); + buf = taosDecodeVariantI32(buf, &pSW->sver); + pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); + if (pSW->pSchema == NULL) { + return NULL; + } + + for (int32_t i = 0; i < pSW->nCols; i++) { + buf = taosDecodeSSchema(buf, &pSW->pSchema[i]); + } + return (void*)buf; +} + +static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SEncoder* pEncoder, const SSchemaWrapper* pSW) { + if (tEncodeI32v(pEncoder, pSW->nCols) < 0) return -1; + if (tEncodeI32v(pEncoder, pSW->sver) < 0) return -1; + for (int32_t i = 0; i < pSW->nCols; i++) { + if (tEncodeSSchema(pEncoder, &pSW->pSchema[i]) < 0) return -1; + } + + return 0; +} + +static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SDecoder* pDecoder, SSchemaWrapper* pSW) { + if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1; + if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1; + + pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); + if (pSW->pSchema == NULL) return -1; + for (int32_t i = 0; i < pSW->nCols; i++) { + if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1; + } + + return 0; +} + STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols); typedef struct { @@ -1873,7 +1976,7 @@ typedef struct SMqHbTopicInfo { int32_t epoch; int64_t topicUid; char name[TSDB_TOPIC_FNAME_LEN]; - SArray* pVgInfo; + SArray* pVgInfo; // SArray } SMqHbTopicInfo; static FORCE_INLINE int32_t taosEncodeSMqHbTopicInfoMsg(void** buf, const SMqHbTopicInfo* pTopicInfo) { @@ -1992,49 +2095,6 @@ static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) return (void*)buf; } -typedef struct { - int8_t reserved; -} SMqRebVgRsp; - -typedef struct { - int64_t leftForVer; - int32_t vgId; - int32_t epoch; - int64_t consumerId; - char topicName[TSDB_TOPIC_FNAME_LEN]; - char cgroup[TSDB_CGROUP_LEN]; - char* sql; - char* physicalPlan; - char* qmsg; -} SMqSetCVgReq; - -static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pReq->leftForVer); - tlen += taosEncodeFixedI32(buf, pReq->vgId); - tlen += taosEncodeFixedI32(buf, pReq->epoch); - tlen += taosEncodeFixedI64(buf, pReq->consumerId); - tlen += taosEncodeString(buf, pReq->topicName); - tlen += taosEncodeString(buf, pReq->cgroup); - tlen += taosEncodeString(buf, pReq->sql); - tlen += taosEncodeString(buf, pReq->physicalPlan); - tlen += taosEncodeString(buf, pReq->qmsg); - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { - buf = taosDecodeFixedI64(buf, &pReq->leftForVer); - buf = taosDecodeFixedI32(buf, &pReq->vgId); - buf = taosDecodeFixedI32(buf, &pReq->epoch); - buf = taosDecodeFixedI64(buf, &pReq->consumerId); - buf = taosDecodeStringTo(buf, pReq->topicName); - buf = taosDecodeStringTo(buf, pReq->cgroup); - buf = taosDecodeString(buf, &pReq->sql); - buf = taosDecodeString(buf, &pReq->physicalPlan); - buf = taosDecodeString(buf, &pReq->qmsg); - return buf; -} - typedef struct { int32_t vgId; int64_t offset; @@ -2056,109 +2116,6 @@ int32_t tDecodeSMqOffset(SDecoder* decoder, SMqOffset* pOffset); int32_t tEncodeSMqCMCommitOffsetReq(SEncoder* encoder, const SMqCMCommitOffsetReq* pReq); int32_t tDecodeSMqCMCommitOffsetReq(SDecoder* decoder, SMqCMCommitOffsetReq* pReq); -static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* pSchemaWrapper) { - SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper)); - if (pSW == NULL) return pSW; - pSW->nCols = pSchemaWrapper->nCols; - pSW->sver = pSchemaWrapper->sver; - pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); - if (pSW->pSchema == NULL) { - taosMemoryFree(pSW); - return NULL; - } - memcpy(pSW->pSchema, pSchemaWrapper->pSchema, pSW->nCols * sizeof(SSchema)); - return pSW; -} - -static FORCE_INLINE void tDeleteSSchemaWrapper(SSchemaWrapper* pSchemaWrapper) { - taosMemoryFree(pSchemaWrapper->pSchema); - taosMemoryFree(pSchemaWrapper); -} - -static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) { - int32_t tlen = 0; - tlen += taosEncodeFixedI8(buf, pSchema->type); - tlen += taosEncodeFixedI8(buf, pSchema->flags); - tlen += taosEncodeFixedI32(buf, pSchema->bytes); - tlen += taosEncodeFixedI16(buf, pSchema->colId); - tlen += taosEncodeString(buf, pSchema->name); - return tlen; -} - -static FORCE_INLINE void* taosDecodeSSchema(const void* buf, SSchema* pSchema) { - buf = taosDecodeFixedI8(buf, &pSchema->type); - buf = taosDecodeFixedI8(buf, &pSchema->flags); - buf = taosDecodeFixedI32(buf, &pSchema->bytes); - buf = taosDecodeFixedI16(buf, &pSchema->colId); - buf = taosDecodeStringTo(buf, pSchema->name); - return (void*)buf; -} - -static FORCE_INLINE int32_t tEncodeSSchema(SEncoder* pEncoder, const SSchema* pSchema) { - if (tEncodeI8(pEncoder, pSchema->type) < 0) return -1; - if (tEncodeI8(pEncoder, pSchema->flags) < 0) return -1; - if (tEncodeI32v(pEncoder, pSchema->bytes) < 0) return -1; - if (tEncodeI16v(pEncoder, pSchema->colId) < 0) return -1; - if (tEncodeCStr(pEncoder, pSchema->name) < 0) return -1; - return 0; -} - -static FORCE_INLINE int32_t tDecodeSSchema(SDecoder* pDecoder, SSchema* pSchema) { - if (tDecodeI8(pDecoder, &pSchema->type) < 0) return -1; - if (tDecodeI8(pDecoder, &pSchema->flags) < 0) return -1; - if (tDecodeI32v(pDecoder, &pSchema->bytes) < 0) return -1; - if (tDecodeI16v(pDecoder, &pSchema->colId) < 0) return -1; - if (tDecodeCStrTo(pDecoder, pSchema->name) < 0) return -1; - return 0; -} - -static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) { - int32_t tlen = 0; - tlen += taosEncodeVariantI32(buf, pSW->nCols); - tlen += taosEncodeVariantI32(buf, pSW->sver); - for (int32_t i = 0; i < pSW->nCols; i++) { - tlen += taosEncodeSSchema(buf, &pSW->pSchema[i]); - } - return tlen; -} - -static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapper* pSW) { - buf = taosDecodeVariantI32(buf, &pSW->nCols); - buf = taosDecodeVariantI32(buf, &pSW->sver); - pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); - if (pSW->pSchema == NULL) { - return NULL; - } - - for (int32_t i = 0; i < pSW->nCols; i++) { - buf = taosDecodeSSchema(buf, &pSW->pSchema[i]); - } - return (void*)buf; -} - -static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SEncoder* pEncoder, const SSchemaWrapper* pSW) { - if (tEncodeI32v(pEncoder, pSW->nCols) < 0) return -1; - if (tEncodeI32v(pEncoder, pSW->sver) < 0) return -1; - for (int32_t i = 0; i < pSW->nCols; i++) { - if (tEncodeSSchema(pEncoder, &pSW->pSchema[i]) < 0) return -1; - } - - return 0; -} - -static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SDecoder* pDecoder, SSchemaWrapper* pSW) { - if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1; - if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1; - - pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); - if (pSW->pSchema == NULL) return -1; - for (int32_t i = 0; i < pSW->nCols; i++) { - if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1; - } - - return 0; -} - typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char stb[TSDB_TABLE_FNAME_LEN]; @@ -2427,6 +2384,21 @@ typedef struct { SEpSet epSet; } SMqSubVgEp; +static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { + int32_t tlen = 0; + tlen += taosEncodeFixedI32(buf, pVgEp->vgId); + tlen += taosEncodeFixedI64(buf, pVgEp->offset); + tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { + buf = taosDecodeFixedI32(buf, &pVgEp->vgId); + buf = taosDecodeFixedI64(buf, &pVgEp->offset); + buf = taosDecodeSEpSet(buf, &pVgEp->epSet); + return buf; +} + typedef struct { char topic[TSDB_TOPIC_FNAME_LEN]; int8_t isSchemaAdaptive; @@ -2434,6 +2406,43 @@ typedef struct { SSchemaWrapper schema; } SMqSubTopicEp; +static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pTopicEp->topic); + tlen += taosEncodeFixedI8(buf, pTopicEp->isSchemaAdaptive); + int32_t sz = taosArrayGetSize(pTopicEp->vgs); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqSubVgEp* pVgEp = (SMqSubVgEp*)taosArrayGet(pTopicEp->vgs, i); + tlen += tEncodeSMqSubVgEp(buf, pVgEp); + } + tlen += taosEncodeSSchemaWrapper(buf, &pTopicEp->schema); + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp) { + buf = taosDecodeStringTo(buf, pTopicEp->topic); + buf = taosDecodeFixedI8(buf, &pTopicEp->isSchemaAdaptive); + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp)); + if (pTopicEp->vgs == NULL) { + return NULL; + } + for (int32_t i = 0; i < sz; i++) { + SMqSubVgEp vgEp; + buf = tDecodeSMqSubVgEp(buf, &vgEp); + taosArrayPush(pTopicEp->vgs, &vgEp); + } + buf = taosDecodeSSchemaWrapper(buf, &pTopicEp->schema); + return buf; +} + +static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { + // taosMemoryFree(pSubTopicEp->schema.pSchema); + taosArrayDestroy(pSubTopicEp->vgs); +} + typedef struct { SMqRspHead head; int64_t reqOffset; @@ -2512,58 +2521,6 @@ typedef struct { SArray* topics; // SArray } SMqAskEpRsp; -static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { - // taosMemoryFree(pSubTopicEp->schema.pSchema); - taosArrayDestroy(pSubTopicEp->vgs); -} - -static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { - int32_t tlen = 0; - tlen += taosEncodeFixedI32(buf, pVgEp->vgId); - tlen += taosEncodeFixedI64(buf, pVgEp->offset); - tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { - buf = taosDecodeFixedI32(buf, &pVgEp->vgId); - buf = taosDecodeFixedI64(buf, &pVgEp->offset); - buf = taosDecodeSEpSet(buf, &pVgEp->epSet); - return buf; -} - -static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pTopicEp->topic); - tlen += taosEncodeFixedI8(buf, pTopicEp->isSchemaAdaptive); - int32_t sz = taosArrayGetSize(pTopicEp->vgs); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqSubVgEp* pVgEp = (SMqSubVgEp*)taosArrayGet(pTopicEp->vgs, i); - tlen += tEncodeSMqSubVgEp(buf, pVgEp); - } - tlen += taosEncodeSSchemaWrapper(buf, &pTopicEp->schema); - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp) { - buf = taosDecodeStringTo(buf, pTopicEp->topic); - buf = taosDecodeFixedI8(buf, &pTopicEp->isSchemaAdaptive); - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp)); - if (pTopicEp->vgs == NULL) { - return NULL; - } - for (int32_t i = 0; i < sz; i++) { - SMqSubVgEp vgEp; - buf = tDecodeSMqSubVgEp(buf, &vgEp); - taosArrayPush(pTopicEp->vgs, &vgEp); - } - buf = taosDecodeSSchemaWrapper(buf, &pTopicEp->schema); - return buf; -} - static FORCE_INLINE int32_t tEncodeSMqAskEpRsp(void** buf, const SMqAskEpRsp* pRsp) { int32_t tlen = 0; // tlen += taosEncodeString(buf, pRsp->cgroup); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 93a950466b..e541c214de 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -208,6 +208,7 @@ int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readN int64_t walGetFirstVer(SWal *); int64_t walGetSnapshotVer(SWal *); int64_t walGetLastVer(SWal *); +int64_t walGetCommittedVer(SWal *); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 2751e0752e..5eb6866387 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -452,6 +452,7 @@ typedef struct { int8_t withSchema; int8_t withTag; SRWLatch lock; + int32_t consumerCnt; int32_t sqlLen; int32_t astLen; char* sql; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 2a81f28edd..5ad4863322 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -787,6 +787,8 @@ static int32_t mndRetrieveSubscribe(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock } } + // do not show for cleared subscription +#if 0 int32_t sz = taosArrayGetSize(pSub->unassignedVgs); for (int32_t i = 0; i < sz; i++) { SMqVgEp *pVgEp = taosArrayGetP(pSub->unassignedVgs, i); @@ -829,6 +831,7 @@ static int32_t mndRetrieveSubscribe(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock numOfRows++; } +#endif taosRUnLockLatch(&pSub->lock); sdbRelease(pSdb, pSub); } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index b62de0e06e..41d4d5f406 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -89,6 +89,8 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SDB_SET_INT8(pRaw, dataPos, pTopic->withTbName, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->withSchema, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->withTag, TOPIC_ENCODE_OVER); + + SDB_SET_INT32(pRaw, dataPos, pTopic->consumerCnt, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->astLen, TOPIC_ENCODE_OVER); @@ -152,6 +154,8 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pTopic->withSchema, TOPIC_DECODE_OVER); SDB_GET_INT8(pRaw, dataPos, &pTopic->withTag, TOPIC_DECODE_OVER); + SDB_GET_INT32(pRaw, dataPos, &pTopic->consumerCnt, TOPIC_DECODE_OVER); + SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); pTopic->sql = taosMemoryCalloc(pTopic->sqlLen, sizeof(char)); if (pTopic->sql == NULL) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6ca60945cd..4b9551b250 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -401,7 +401,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { fetchOffset = walGetFirstVer(pTq->pWal); } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) { - fetchOffset = walGetLastVer(pTq->pWal); + fetchOffset = walGetCommittedVer(pTq->pWal); } else { fetchOffset = pReq->currentOffset + 1; } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 3e6663f464..9aa848a7bb 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -25,6 +25,8 @@ int64_t FORCE_INLINE walGetSnaphostVer(SWal* pWal) { return pWal->vers.snapshotV int64_t FORCE_INLINE walGetLastVer(SWal* pWal) { return pWal->vers.lastVer; } +int64_t FORCE_INLINE walGetCommittedVer(SWal* pWal) { return pWal->vers.commitVer; } + static FORCE_INLINE int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index e14515286e..0cfe75bf33 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -143,7 +143,11 @@ void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity) { pRead->capa int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead) { int32_t code; + // TODO: valid ver + if (ver > pRead->pWal->vers.commitVer) { + return -1; + } if (pRead->curVersion != ver) { code = walReadSeekVer(pRead, ver); From a816e3034e4f190fa29db1e9ae712a356972cb36 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 11 May 2022 15:18:26 +0800 Subject: [PATCH 22/94] fix(os): Ubuntu 18.04 timezone error. --- source/os/src/osTimezone.c | 194 +++++++++++++++++++------------------ 1 file changed, 98 insertions(+), 96 deletions(-) diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 06ec7e9464..872d8e740c 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -172,106 +172,108 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { -timezone / 3600); #else - if (taosCheckExistFile("/etc/timezone")) { - /* - * NOTE: do not remove it. - * Enforce set the correct daylight saving time(DST) flag according - * to current time - */ - time_t tx1 = taosGetTimestampSec(); - struct tm tm1; - taosLocalTime(&tx1, &tm1); - /* load time zone string from /etc/timezone */ - // FILE *f = fopen("/etc/timezone", "r"); - errno = 0; - TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ); - char buf[68] = {0}; - if (pFile != NULL) { - int len = taosReadFile(pFile, buf, 64); - if (len < 64 && taosGetErrorFile(pFile)) { - taosCloseFile(&pFile); - printf("read /etc/timezone error, reason:%s", strerror(errno)); - return; - } - - taosCloseFile(&pFile); - - buf[sizeof(buf) - 1] = 0; - char *lineEnd = strstr(buf, "\n"); - if (lineEnd != NULL) { - *lineEnd = 0; - } - - // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables - if (strlen(buf) > 0) { - setenv("TZ", buf, 1); - } - } - // get and set default timezone - tzset(); - /* - * get CURRENT time zone. - * system current time zone is affected by daylight saving time(DST) - * - * e.g., the local time zone of London in DST is GMT+01:00, - * otherwise is GMT+00:00 - */ - int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; - *tsTimezone = tz; - tz += daylight; - - /* - * format example: - * - * Asia/Shanghai (CST, +0800) - * Europe/London (BST, +0100) - */ - snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); - } else { - char buf[4096] = {0}; - char *tz = NULL; - { - int n = readlink("/etc/localtime", buf, sizeof(buf)); - if (n < 0) { - printf("read /etc/localtime error, reason:%s", strerror(errno)); - return; - } - buf[n] = '\0'; - for (int i = n - 1; i >= 0; --i) { - if (buf[i] == '/') { - if (tz) { - tz = buf + i + 1; - break; + char buf[4096] = {0}; + char *tz = NULL; + { + int n = readlink("/etc/localtime", buf, sizeof(buf)); + if (n < 0) { + printf("read /etc/localtime error, reason:%s", strerror(errno)); + + if (taosCheckExistFile("/etc/timezone")) { + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = taosGetTimestampSec(); + struct tm tm1; + taosLocalTime(&tx1, &tm1); + /* load time zone string from /etc/timezone */ + // FILE *f = fopen("/etc/timezone", "r"); + errno = 0; + TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ); + char buf[68] = {0}; + if (pFile != NULL) { + int len = taosReadFile(pFile, buf, 64); + if (len < 64 && taosGetErrorFile(pFile)) { + taosCloseFile(&pFile); + printf("read /etc/timezone error, reason:%s", strerror(errno)); + return; } - tz = buf + i + 1; - } - } - if (!tz || 0 == strchr(tz, '/')) { - printf("parsing /etc/localtime failed"); - return; - } - setenv("TZ", tz, 1); - tzset(); + taosCloseFile(&pFile); + + buf[sizeof(buf) - 1] = 0; + char *lineEnd = strstr(buf, "\n"); + if (lineEnd != NULL) { + *lineEnd = 0; + } + + // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables + if (strlen(buf) > 0) { + setenv("TZ", buf, 1); + } + } + // get and set default timezone + tzset(); + /* + * get CURRENT time zone. + * system current time zone is affected by daylight saving time(DST) + * + * e.g., the local time zone of London in DST is GMT+01:00, + * otherwise is GMT+00:00 + */ + int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; + *tsTimezone = tz; + tz += daylight; + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + } else { + printf("There is not /etc/timezone.\n"); + } + return; + } + buf[n] = '\0'; + for (int i = n - 1; i >= 0; --i) { + if (buf[i] == '/') { + if (tz) { + tz = buf + i + 1; + break; + } + tz = buf + i + 1; + } + } + if (!tz || 0 == strchr(tz, '/')) { + printf("parsing /etc/localtime failed"); + return; } - /* - * NOTE: do not remove it. - * Enforce set the correct daylight saving time(DST) flag according - * to current time - */ - time_t tx1 = taosGetTimestampSec(); - struct tm tm1; - taosLocalTime(&tx1, &tm1); - - /* - * format example: - * - * Asia/Shanghai (CST, +0800) - * Europe/London (BST, +0100) - */ - snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], - -timezone / 3600); + setenv("TZ", tz, 1); + tzset(); } + + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = taosGetTimestampSec(); + struct tm tm1; + taosLocalTime(&tx1, &tm1); + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], + -timezone / 3600); #endif } From 38932585ba9b270dd52c986b3decb09bcbb90bbc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 May 2022 07:41:51 +0000 Subject: [PATCH 23/94] refact more --- include/common/tdataformat.h | 13 ++--- source/common/src/tdataformat.c | 99 +++++++++++++++++++++++++++------ 2 files changed, 89 insertions(+), 23 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index e270e09471..2e8a214a9d 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -33,9 +33,6 @@ typedef struct STSRow2 STSRow2; typedef struct STSRowBuilder STSRowBuilder; typedef struct SKVIdx SKVIdx; -#define TD_TP_ROW 0x0U -#define TD_KV_ROW 0x1U - // STSchema // STSRow2 @@ -48,8 +45,8 @@ void tTSchemaDestroy(STSchema *pTSchema); // STSRowBuilder int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols); -int32_t tTSRowBuilderClear(STSRowBuilder *pBuilder); -int32_t tTSRowBuilderReset(STSRowBuilder *pBuilder); +void tTSRowBuilderClear(STSRowBuilder *pBuilder); +void tTSRowBuilderReset(STSRowBuilder *pBuilder); int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData); int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow); @@ -85,11 +82,13 @@ struct STSRow2 { struct STSRowBuilder { STColumn *pTColumn; STSchema *pTSchema; + int32_t szKVBuf; + uint8_t *pKVBuf; + int32_t szTPBuf; + uint8_t *pTPBuf; int32_t nCols; int32_t kvVLen; - uint8_t *pKV; int32_t tpVLen; - uint8_t *pTuple; STSRow2 row; }; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index c39ad3821f..ad020fe7d9 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -19,6 +19,8 @@ #include "tdatablock.h" #include "tlog.h" +#define TD_KV_ROW 0x1U + struct SKVIdx { int32_t cid; int32_t offset; @@ -86,16 +88,46 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema * void tTSchemaDestroy(STSchema *pTSchema) { taosMemoryFree(pTSchema); } int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) { - // TODO + int32_t kvBufLen; + int32_t tpBufLen; + uint8_t *p; + + if (tTSchemaCreate(sver, pSchema, nCols, &pBuilder->pTSchema) < 0) return -1; + + kvBufLen = sizeof(SKVIdx) * nCols + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; + tpBufLen = pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; + + if (pBuilder->szKVBuf < kvBufLen) { + p = taosMemoryRealloc(pBuilder->pKVBuf, kvBufLen); + if (p == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pBuilder->pKVBuf = p; + pBuilder->szKVBuf = kvBufLen; + } + + if (pBuilder->szTPBuf < tpBufLen) { + p = taosMemoryRealloc(pBuilder->pTPBuf, tpBufLen); + if (p == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pBuilder->pTPBuf = p; + pBuilder->szTPBuf = tpBufLen; + } + + tTSRowBuilderReset(pBuilder); + return 0; } -int32_t tTSRowBuilderClear(STSRowBuilder *pBuilder) { - // TODO - return 0; +void tTSRowBuilderClear(STSRowBuilder *pBuilder) { + taosMemoryFree(pBuilder->pKVBuf); + taosMemoryFree(pBuilder->pTPBuf); } -int32_t tTSRowBuilderReset(STSRowBuilder *pBuilder) { +void tTSRowBuilderReset(STSRowBuilder *pBuilder) { for (int32_t iCol = pBuilder->pTSchema->numOfCols - 1; iCol >= 0; iCol--) { pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; @@ -106,19 +138,29 @@ int32_t tTSRowBuilderReset(STSRowBuilder *pBuilder) { pBuilder->kvVLen = 0; pBuilder->tpVLen = 0; pBuilder->row.flags = 0; - - return 0; } int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData) { + int32_t iCol; + uint8_t *p; + + // search column if (pBuilder->pTColumn->colId < cid) { - // right search + iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1; + for (; iCol < pBuilder->pTSchema->numOfCols; iCol++) { + pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; + if (pBuilder->pTColumn->colId == cid) break; + } } else if (pBuilder->pTColumn->colId > cid) { - // left search + iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) - 1; + for (; iCol >= 0; iCol--) { + pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; + if (pBuilder->pTColumn->colId == cid) break; + } } - // check if val is set already - if (pBuilder->pTColumn->flags & COL_VAL_SET) { + // check + if (pBuilder->pTColumn->colId != cid || pBuilder->pTColumn->flags & COL_VAL_SET) { return -1; } @@ -128,7 +170,32 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD pBuilder->row.ts = *(TSKEY *)pData; } else { if (pData) { - // set val + // ASSERT(!IS_NULL(pData)); + + // set tuple data + p = pBuilder->pTPBuf + pBuilder->pTColumn->offset; + if (IS_VAR_DATA_TYPE(pBuilder->pTColumn->type)) { + *(int32_t *)p = pBuilder->tpVLen; + + // encode the variant-length data + p = pBuilder->pTPBuf + pBuilder->pTSchema->flen + pBuilder->tpVLen; + pBuilder->tpVLen += tPutBinary(p, pData, nData); + } else { + memcpy(p, pData, nData); + } + + // set kv data + p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols; + ((SKVIdx *)p)->cid = cid; + ((SKVIdx *)p)->offset = pBuilder->kvVLen; + + p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols + pBuilder->kvVLen; + if (IS_VAR_DATA_TYPE(pBuilder->pTColumn->type)) { + pBuilder->kvVLen += tPutBinary(p, pData, nData); + } else { + memcpy(p, pData, nData); + pBuilder->kvVLen += nData; + } } else { // set NULL val } @@ -149,18 +216,18 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { pBuilder->row.flags |= TD_KV_ROW; pBuilder->row.ncols = pBuilder->nCols; pBuilder->row.nData = pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen; - pBuilder->row.pData = pBuilder->pKV; + pBuilder->row.pData = pBuilder->pKVBuf; if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) { - memmove(pBuilder->pKV + sizeof(SKVIdx) * pBuilder->nCols, - pBuilder->pKV + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols, pBuilder->kvVLen); + memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols, + pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols, pBuilder->kvVLen); } } else { // encode as TD_TUPLE_ROW pBuilder->row.flags &= (~TD_KV_ROW); pBuilder->row.sver = pBuilder->pTSchema->version; pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen; - pBuilder->row.pData = pBuilder->pTuple; + pBuilder->row.pData = pBuilder->pTPBuf; if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) { // set non-set cols as None From 2b8e3a91a39d4037453e810b3c2c9563c36cdbbf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 May 2022 07:41:55 +0000 Subject: [PATCH 24/94] refact more --- include/util/tencode.h | 49 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/include/util/tencode.h b/include/util/tencode.h index 8504aec5ea..2a43d7934f 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -457,6 +457,55 @@ static FORCE_INLINE void* tDecoderMalloc(SDecoder* pCoder, int32_t size) { return p; } +static FORCE_INLINE int32_t tPutBinary(uint8_t* p, const uint8_t* pData, uint32_t nData) { + int n = 0; + uint32_t v = nData; + + for (;;) { + if (v <= 0x7f) { + if (p) p[n] = v; + n++; + break; + } + + if (p) p[n] = (v & 0x7f) | 0x80; + n++; + v >>= 7; + } + + if (p) { + memcpy(p + n, pData, nData); + } + n += nData; + + return n; +} + +static FORCE_INLINE int32_t tGetBinary(const uint8_t* p, const uint8_t** ppData, uint32_t* nData) { + int32_t n = 0; + uint32_t tv = 0; + uint32_t t; + + for (;;) { + if (p[n] <= 0x7f) { + t = p[n]; + tv |= (t << (7 * n)); + n++; + break; + } + + t = p[n] & 0x7f; + tv |= (t << (7 * n)); + n++; + } + + if (nData) *nData = n; + if (ppData) *ppData = p + n; + + n += tv; + return n; +} + #ifdef __cplusplus } #endif From 977c3fe4008ae9eef78d3d1f3031e440479444ea Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 11 May 2022 15:42:31 +0800 Subject: [PATCH 25/94] feat(query): add state_duration function --- source/libs/function/inc/builtinsimpl.h | 1 + source/libs/function/src/builtins.c | 39 +++++++++++++++++- source/libs/function/src/builtinsimpl.c | 54 ++++++++++++++++++++++++- 3 files changed, 91 insertions(+), 3 deletions(-) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 607ab00e7d..e3b7127efe 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -92,6 +92,7 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getStateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool stateFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t stateCountFunction(SqlFunctionCtx* pCtx); +int32_t stateDurationFunction(SqlFunctionCtx* pCtx); bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index a94777c7f2..07ad0f7d1f 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -263,7 +263,7 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l return TSDB_CODE_SUCCESS; } -static int32_t translateState(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { +static int32_t translateStateCount(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { if (3 != LIST_LENGTH(pFunc->pParameterList)) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } @@ -283,6 +283,31 @@ static int32_t translateState(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return TSDB_CODE_SUCCESS; } +static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + int32_t paraNum = LIST_LENGTH(pFunc->pParameterList); + if (3 != paraNum && 4 != paraNum) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_NUMERIC_TYPE(colType)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type != TSDB_DATA_TYPE_BINARY || + (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_BIGINT && + ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_DOUBLE)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + if (paraNum == 4 && ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type != TSDB_DATA_TYPE_BIGINT) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT }; + return TSDB_CODE_SUCCESS; +} + static int32_t translateLastRow(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // todo return TSDB_CODE_SUCCESS; @@ -701,12 +726,22 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "state_count", .type = FUNCTION_TYPE_STATE_COUNT, .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC, - .translateFunc = translateState, + .translateFunc = translateStateCount, .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, .processFunc = stateCountFunction, .finalizeFunc = NULL }, + { + .name = "state_duration", + .type = FUNCTION_TYPE_STATE_DURATION, + .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateStateDuration, + .getEnvFunc = getStateFuncEnv, + .initFunc = functionSetup, + .processFunc = stateDurationFunction, + .finalizeFunc = NULL + }, { .name = "abs", .type = FUNCTION_TYPE_ABS, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index c255c55d4e..efc2992075 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -125,7 +125,10 @@ typedef enum { } EHistoBinType; typedef struct SStateInfo { - int64_t count; + union { + int64_t count; + int64_t durationStart; + }; } SStateInfo; typedef enum { @@ -2841,3 +2844,52 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) { return numOfElems; } + +int32_t stateDurationFunction(SqlFunctionCtx* pCtx) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SStateInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + SInputColumnInfoData* pInput = &pCtx->input; + TSKEY* tsList = (int64_t*)pInput->pPTS->pData; + + SColumnInfoData* pInputCol = pInput->pData[0]; + SColumnInfoData* pTsOutput = pCtx->pTsOutput; + + int32_t numOfElems = 0; + SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; + + //TODO: process timeUnit for different db precisions + int32_t timeUnit = 1000; + if (pCtx->numOfParams == 5) { //TODO: param number incorrect + timeUnit = pCtx->param[3].param.i; + } + + int8_t op = getStateOpType(varDataVal(pCtx->param[1].param.pz)); + if (STATE_OPER_INVALID == op) { + return 0; + } + + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + numOfElems++; + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + colDataAppendNULL(pOutput, i); + continue; + } + + bool ret = checkStateOp(op, pInputCol, i, pCtx->param[2].param); + int64_t output = -1; + if (ret) { + if (pInfo->durationStart == 0) { + output = 0; + pInfo->durationStart = tsList[i]; + } else { + output = (tsList[i] - pInfo->durationStart) / timeUnit; + } + } else { + pInfo->durationStart = 0; + } + colDataAppend(pOutput, i, (char *)&output, false); + } + + return numOfElems; +} From 7c41d52967daaef7ac5ddcf673725bdc3bb4b954 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 11 May 2022 15:52:31 +0800 Subject: [PATCH 26/94] enh(sync): add error log, linux api error --- source/libs/sync/src/syncRaftLog.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 66c85b9cbb..3ee952fcdf 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -60,7 +60,9 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { if (code != 0) { int32_t err = terrno; const char *errStr = tstrerror(err); - sError("walWriteWithSyncInfo error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); + int32_t linuxErr = errno; + const char *linuxErrMsg = strerror(errno); + sError("walWriteWithSyncInfo error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, linuxErr, linuxErrMsg); ASSERT(0); } //assert(code == 0); @@ -79,7 +81,9 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { if (code != 0) { int32_t err = terrno; const char *errStr = tstrerror(err); - sError("walReadWithHandle error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); + int32_t linuxErr = errno; + const char *linuxErrMsg = strerror(errno); + sError("walReadWithHandle error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, linuxErr, linuxErrMsg); ASSERT(0); } //assert(walReadWithHandle(pWalHandle, index) == 0); @@ -113,7 +117,9 @@ int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { if (code != 0) { int32_t err = terrno; const char *errStr = tstrerror(err); - sError("walRollback error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); + int32_t linuxErr = errno; + const char *linuxErrMsg = strerror(errno); + sError("walRollback error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, linuxErr, linuxErrMsg); ASSERT(0); } return 0; // to avoid compiler error @@ -144,7 +150,9 @@ int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { if (code != 0) { int32_t err = terrno; const char *errStr = tstrerror(err); - sError("walCommit error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); + int32_t linuxErr = errno; + const char *linuxErrMsg = strerror(errno); + sError("walCommit error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, linuxErr, linuxErrMsg); ASSERT(0); } return 0; // to avoid compiler error From 9fcc676e3f8801c553f888db528a8f9c6f9a8e33 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 11 May 2022 16:04:33 +0800 Subject: [PATCH 27/94] stmt auto create table --- source/client/src/clientStmt.c | 6 +++++- source/libs/parser/src/parInsert.c | 2 +- tests/script/api/batchprepare.c | 12 +++++++----- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 3a2363f882..2dff1c458c 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -339,6 +339,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { STableDataBlocks *pBlockInExec = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (pBlockInExec) { + pStmt->bInfo.needParse = false; pStmt->bInfo.inExecCache = true; if (pStmt->sql.autoCreateTbl) { @@ -364,7 +365,10 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pStmt->bInfo.tbSuid, sizeof(pStmt->bInfo.tbSuid)); if (pCache) { pStmt->bInfo.needParse = false; - + pStmt->exec.autoCreateTbl = true; + + pStmt->bInfo.tbUid = 0; + STableDataBlocks* pNewBlock = NULL; STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataBlock, &pNewBlock, 0)); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index ad0319cf69..c38d577abf 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1393,7 +1393,6 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in tdSRowPrint(row, pSTSchema, __func__); taosMemoryFree(pSTSchema); #endif - pDataBlock->size += extendedRowSize; } @@ -1480,6 +1479,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu taosMemoryFree(pSTSchema); } #endif + } if (rowEnd) { diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 4423ee3bc8..9f9ed88b89 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -233,9 +233,9 @@ CaseCtrl gCaseCtrl = { .printRes = false, .runTimes = 0, .caseIdx = -1, - .caseNum = -1, - .caseRunIdx = -1, - .caseRunNum = -1, + .caseNum = 1, + .caseRunIdx = 20, + .caseRunNum = 1, }; #endif @@ -2988,7 +2988,7 @@ void prepareCheckResultImpl(TAOS * taos, char *tname, bool printr, int expec } } else { printf("!!!expect rows %d mis-match rows %d fetched from %s\n", expected, rows, tname); - //exit(1); + exit(1); } } @@ -3465,11 +3465,13 @@ void* runCaseList(TAOS *taos) { } void runAll(TAOS *taos) { +#if 0 strcpy(gCaseCtrl.caseCatalog, "Normal Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); runCaseList(taos); +#endif -#if 0 +#if 1 strcpy(gCaseCtrl.caseCatalog, "Auto Create Table Test"); gCaseCtrl.autoCreateTbl = true; printf("%s Begin\n", gCaseCtrl.caseCatalog); From 0e754415459974569e43d66bf71edef70204332d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 May 2022 08:10:17 +0000 Subject: [PATCH 28/94] fix ctest --- source/dnode/mnode/impl/src/mndDb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 645f2ff0e7..c84cc10050 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -412,8 +412,8 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->numOfVgroups < 0) pCfg->numOfVgroups = TSDB_DEFAULT_VN_PER_DB; if (pCfg->numOfStables < 0) pCfg->numOfStables = TSDB_DEFAULT_DB_SINGLE_STABLE; if (pCfg->buffer < 0) pCfg->buffer = TSDB_DEFAULT_BUFFER_PER_VNODE; - if (pCfg->pageSize < 0) pCfg->pageSize = TSDB_DEFAULT_PAGES_PER_VNODE; - if (pCfg->pages < 0) pCfg->pages = TSDB_MAX_PAGESIZE_PER_VNODE; + if (pCfg->pageSize < 0) pCfg->pageSize = TSDB_DEFAULT_PAGESIZE_PER_VNODE; + if (pCfg->pages < 0) pCfg->pages = TSDB_DEFAULT_PAGES_PER_VNODE; if (pCfg->daysPerFile < 0) pCfg->daysPerFile = TSDB_DEFAULT_DURATION_PER_FILE; if (pCfg->daysToKeep0 < 0) pCfg->daysToKeep0 = TSDB_DEFAULT_KEEP; if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = pCfg->daysToKeep0; From ce5e6cdbcb16df2f0a7643903741a236ff156a57 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 May 2022 08:26:18 +0000 Subject: [PATCH 29/94] fix drop table problem --- source/dnode/vnode/src/vnd/vnodeSvr.c | 18 +++++++++++------- source/libs/tdb/src/db/tdbBtree.c | 1 + 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 0082ca0802..630a7ffd43 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -392,7 +392,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, tEncoderClear(&encoder); _exit: - taosArrayClear(rsp.pArray); + taosArrayDestroy(rsp.pArray); tDecoderClear(&decoder); tEncoderClear(&encoder); return rcode; @@ -454,6 +454,7 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in SVDropTbBatchReq req = {0}; SVDropTbBatchRsp rsp = {0}; SDecoder decoder = {0}; + SEncoder encoder = {0}; int ret; pRsp->msgType = TDMT_VND_DROP_TABLE_RSP; @@ -471,7 +472,7 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in } // process req - rsp.pArray = taosArrayInit(sizeof(SVDropTbRsp), req.nReqs); + rsp.pArray = taosArrayInit(req.nReqs, sizeof(SVDropTbRsp)); for (int iReq = 0; iReq < req.nReqs; iReq++) { SVDropTbReq *pDropTbReq = req.pReqs + iReq; SVDropTbRsp dropTbRsp = {0}; @@ -493,11 +494,15 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in _exit: tDecoderClear(&decoder); - // encode rsp (TODO) + tEncodeSize(tEncodeSVDropTbBatchRsp, &rsp, pRsp->contLen, ret); + pRsp->pCont = rpcMallocCont(pRsp->contLen); + tEncoderInit(&encoder, pRsp->pCont, pRsp->contLen); + tEncodeSVDropTbBatchRsp(&encoder, &rsp); + tEncoderClear(&encoder); return 0; } -static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char* tags) { +static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char *tags) { ASSERT(pMsg != NULL); SSubmitMsgIter msgIter = {0}; SMeta *pMeta = pVnode->pMeta; @@ -518,11 +523,11 @@ static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char taosMemoryFreeClear(pSchema); } pSchema = metaGetTbTSchema(pMeta, msgIter.suid, 0); // TODO: use the real schema - if(pSchema) { + if (pSchema) { suid = msgIter.suid; } } - if(!pSchema) { + if (!pSchema) { printf("%s:%d no valid schema\n", tags, __LINE__); continue; } @@ -596,7 +601,6 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in } rsp.affectedRows += nRows; - } _exit: diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index cf7dd50103..869b289931 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1528,6 +1528,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int } } + tdbOsFree(pBuf); return 0; } From 7360d77a69db012cdd97a772cc3766bc516e8a78 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 May 2022 08:39:32 +0000 Subject: [PATCH 30/94] fix invalid free --- source/libs/tdb/src/db/tdbBtree.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 869b289931..cf7dd50103 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1528,7 +1528,6 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int } } - tdbOsFree(pBuf); return 0; } From c0bd8a2f831cb2fbd68c97f8bf8877a77ae50aa3 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 11 May 2022 16:41:49 +0800 Subject: [PATCH 31/94] fix: calculate col distance --- source/common/src/trow.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 44bcd72a33..d1516403c1 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -1159,7 +1159,7 @@ bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, #ifdef TD_SUPPORT_BITMAP int16_t colIdx = -1; - if (pKvIdx) colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pKvIdx) / sizeof(SKvRowIdx); + if (pKvIdx) colIdx = POINTER_DISTANCE(pKvIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx); if (tdGetBitmapValType(pIter->pBitmap, colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { pVal->valType = TD_VTYPE_NONE; } @@ -1226,7 +1226,7 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell compareKvRowColId, TD_EQ); #ifdef TD_SUPPORT_BITMAP if (pIdx) { - colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pIdx) / sizeof(SKvRowIdx); + colIdx = POINTER_DISTANCE(pIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx); } #endif tdGetKvRowValOfCol(pVal, pRow, pIter->pBitmap, pIdx ? pIdx->offset : -1, colIdx); From a1e4ac8910410dd4aa07b9f0f232e174b788d659 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 11 May 2022 17:37:30 +0800 Subject: [PATCH 32/94] test: add test case for tmq --- tests/system-test/7-tmq/basic5.py | 141 ++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 tests/system-test/7-tmq/basic5.py diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py new file mode 100644 index 0000000000..99aa4e72aa --- /dev/null +++ b/tests/system-test/7-tmq/basic5.py @@ -0,0 +1,141 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + hostname = socket.gethostname() + rpcDebugFlagVal = '143' + clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + + updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + + print ("===================: ", updatecfgDict) + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def create_tables(self,dbName,vgroups,stbName,ctbNum,rowsPerTbl): + tdSql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tdSql.execute("use %s" %dbName) + tdSql.execute("create table %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) + pre_create = "create table" + sql = pre_create + #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + for i in range(ctbNum): + sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1) + if (i > 0) and (i%100 == 0): + tdSql.execute(sql) + sql = pre_create + if sql != pre_create: + tdSql.execute(sql) + + tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum)) + return + + def insert_data(self,dbName,stbName,ctbNum,rowsPerTbl,startTs): + tdLog.debug("start to insert data ............") + tdSql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + for i in range(ctbNum): + sql += " %s_%d values "%(stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + if (j > 0) and (j%2000 == 0): + tdSql.execute(sql) + sql = "insert into %s_%d values " %(stbName,i) + #end sql + if sql != pre_insert: + # print(sql) + print("sql:%s"%sql) + tdSql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareEnv(self, **parameterDict): + print ("input parameters:") + print (parameterDict) + self.create_tables(parameterDict["dbName"],\ + parameterDict["vgroups"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"]) + + self.insert_data(parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["startTs"]) + return + + def run(self): + tdSql.prepare() + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + tdLog.printNoPrefix("======== test scenario 1: ") + tdLog.info("step 1: create database, stb, ctb and insert data") + # create and start thread + parameterDict = {'dbName': 'db', \ + 'vgroups': 1, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + # wait for data ready + prepareEnvThread.join() + + tdLog.printNoPrefix("======== test scenario 2: ") + + + tdLog.printNoPrefix("======== test scenario 3: ") + + #os.system('pkill tmq_sim') + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 17bbd59a824649f4a63e374943147d488c8c619c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 May 2022 18:00:42 +0800 Subject: [PATCH 33/94] enh(index): update index filter --- source/libs/executor/src/indexoperator.c | 110 ++++++++++++++++++----- 1 file changed, 90 insertions(+), 20 deletions(-) diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 86a28605b2..fe30ebb2ea 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -21,7 +21,9 @@ typedef struct SIFCtx { int32_t code; - SHashObj *pRes; /* element is SScalarParam */ + SHashObj *pRes; /* element is SScalarParam */ + bool noExec; // true: just iterate condition tree, and add hint to executor plan + // SIdxFltStatus st; } SIFCtx; #define SIF_ERR_RET(c) \ @@ -55,11 +57,12 @@ typedef struct SIFParam { SArray *result; char * condValue; - uint8_t colValType; - col_id_t colId; - int64_t suid; // add later - char dbName[TSDB_DB_NAME_LEN]; - char colName[TSDB_COL_NAME_LEN]; + SIdxFltStatus status; + uint8_t colValType; + col_id_t colId; + int64_t suid; // add later + char dbName[TSDB_DB_NAME_LEN]; + char colName[TSDB_COL_NAME_LEN]; } SIFParam; static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { @@ -82,6 +85,9 @@ static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { } typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output); + +static sif_func_t sifNullFunc = NULL; +// typedef struct SIFWalkParm // construct tag filter operator later static void destroyTagFilterOperatorInfo(void *param) { STagFilterOperatorInfo *pInfo = (STagFilterOperatorInfo *)param; @@ -114,6 +120,24 @@ static int32_t sifValidateColumn(SColumnNode *cn) { return TSDB_CODE_SUCCESS; } +static SIdxFltStatus sifMergeCond(ELogicConditionType type, SIdxFltStatus ls, SIdxFltStatus rs) { + // enh rule later + if (type == LOGIC_COND_TYPE_AND) { + if (ls == SFLT_NOT_INDEX || rs == SFLT_NOT_INDEX) { + if (ls == SFLT_NOT_INDEX) + return rs; + else + return ls; + } + return SFLT_COARSE_INDEX; + } else if (type == LOGIC_COND_TYPE_OR) { + return SFLT_COARSE_INDEX; + } else if (type == LOGIC_COND_TYPE_NOT) { + return SFLT_NOT_INDEX; + } + return SFLT_NOT_INDEX; +} + static int32_t sifGetValueFromNode(SNode *node, char **value) { // covert data From snode; SValueNode *vn = (SValueNode *)node; @@ -257,11 +281,11 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP if (tm == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } - SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); EIndexQueryType qtype = 0; SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); + SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); indexMultiTermQueryAdd(mtm, tm, qtype); int ret = indexSearch(NULL, mtm, output->result); indexMultiTermQueryDestroy(mtm); @@ -319,6 +343,7 @@ static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output int id = OP_TYPE_NMATCH; return sifDoIndex(left, right, id, output); } + static int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) { // add more except return TSDB_CODE_QRY_INVALID_INPUT; @@ -352,9 +377,9 @@ static sif_func_t sifGetOperFn(int32_t funcId) { case OP_TYPE_NMATCH: return sifNotMatchFunc; default: - return sifDefaultFunc; + return sifNullFunc; } - return sifDefaultFunc; + return sifNullFunc; } static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { int32_t code = 0; @@ -367,6 +392,11 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); sif_func_t operFn = sifGetOperFn(node->opType); + if (ctx->noExec && operFn == NULL) { + output->status = SFLT_NOT_INDEX; + } else { + output->status = SFLT_ACCURATE_INDEX; + } return operFn(¶ms[0], nParam > 1 ? ¶ms[1] : NULL, output); _return: @@ -385,14 +415,20 @@ static int32_t sifExecLogic(SLogicConditionNode *node, SIFCtx *ctx, SIFParam *ou SIFParam *params = NULL; SIF_ERR_RET(sifInitParamList(¶ms, node->pParameterList, ctx)); - for (int32_t m = 0; m < node->pParameterList->length; m++) { - // add impl later - if (node->condType == LOGIC_COND_TYPE_AND) { - taosArrayAddAll(output->result, params[m].result); - } else if (node->condType == LOGIC_COND_TYPE_OR) { - taosArrayAddAll(output->result, params[m].result); - } else if (node->condType == LOGIC_COND_TYPE_NOT) { - taosArrayAddAll(output->result, params[m].result); + if (ctx->noExec == false) { + for (int32_t m = 0; m < node->pParameterList->length; m++) { + // add impl later + if (node->condType == LOGIC_COND_TYPE_AND) { + taosArrayAddAll(output->result, params[m].result); + } else if (node->condType == LOGIC_COND_TYPE_OR) { + taosArrayAddAll(output->result, params[m].result); + } else if (node->condType == LOGIC_COND_TYPE_NOT) { + taosArrayAddAll(output->result, params[m].result); + } + } + } else { + for (int32_t m = 0; m < node->pParameterList->length; m++) { + output->status = sifMergeCond(node->condType, output->status, params[m].status); } } _return: @@ -486,7 +522,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { return TSDB_CODE_QRY_INVALID_INPUT; } int32_t code = 0; - SIFCtx ctx = {.code = 0}; + SIFCtx ctx = {.code = 0, .noExec = false}; ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { qError("index-filter failed to taosHashInit"); @@ -510,6 +546,36 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { SIF_RET(code); } +static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) { + int32_t code = TSDB_CODE_SUCCESS; + if (pNode == NULL) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SIFCtx ctx = {.code = 0, .noExec = true}; + ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (NULL == ctx.pRes) { + qError("index-filter failed to taosHashInit"); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + nodesWalkExprPostOrder(pNode, sifCalcWalker, &ctx); + + SIF_ERR_RET(ctx.code); + + SIFParam *res = (SIFParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); + if (res == NULL) { + qError("no valid res in hash, node:(%p), type(%d)", (void *)&pNode, nodeType(pNode)); + SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + *status = res->status; + + sifFreeParam(res); + taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); + + SIF_RET(code); +} + int32_t doFilterTag(const SNode *pFilterNode, SArray *result) { if (pFilterNode == NULL) { return TSDB_CODE_SUCCESS; @@ -528,10 +594,14 @@ int32_t doFilterTag(const SNode *pFilterNode, SArray *result) { } SIdxFltStatus idxGetFltStatus(SNode *pFilterNode) { + SIdxFltStatus st = SFLT_NOT_INDEX; if (pFilterNode == NULL) { return SFLT_NOT_INDEX; } + SFilterInfo *filter = NULL; + // todo move to the initialization function + SIF_ERR_RET(filterInitFromNode((SNode *)pFilterNode, &filter, 0)); - // impl later - return SFLT_ACCURATE_INDEX; + SIF_ERR_RET(sifGetFltHint((SNode *)pFilterNode, &st)); + return st; } From 752e105c0aec51ac84c1904461a2f6793bd08cad Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 11 May 2022 18:24:05 +0800 Subject: [PATCH 34/94] fix(os): windows compile TDinternal. --- source/client/src/clientSml.c | 6 +++--- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 4 ++-- source/os/src/osMemory.c | 3 ++- source/util/src/tcache.c | 3 ++- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index c82f54f32b..b5d09d6fd5 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -402,7 +402,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_INVALID_STB) { - SSchemaAction schemaAction = {.action = SCHEMA_ACTION_CREATE_STABLE}; + SSchemaAction schemaAction = { SCHEMA_ACTION_CREATE_STABLE, 0}; memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); schemaAction.createSTable.tags = sTableData->tags; schemaAction.createSTable.fields = sTableData->cols; @@ -1505,8 +1505,8 @@ static int32_t smlParseLine(SSmlHandle* info, const char* sql) { tinfo->sTableName = elements.measure; tinfo->sTableNameLen = elements.measureLen; - RandTableName rName = {.tags=tinfo->tags, .sTableName=tinfo->sTableName, .sTableNameLen=tinfo->sTableNameLen, - .childTableName=tinfo->childTableName}; + RandTableName rName = { tinfo->tags, tinfo->sTableName, tinfo->sTableNameLen, + tinfo->childTableName, 0 }; buildChildTableName(&rName); tinfo->uid = rName.uid; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 3168ff53f6..2acca738fb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -38,7 +38,7 @@ struct SMemTable { struct SMemSkipListNode { int8_t level; - SMemSkipListNode *forwards[]; + SMemSkipListNode *forwards[1]; // Windows does not allow 0 }; struct SMemSkipList { @@ -46,7 +46,7 @@ struct SMemSkipList { int8_t maxLevel; int8_t level; int32_t size; - SMemSkipListNode pHead[]; + SMemSkipListNode pHead[1]; // Windows does not allow 0 }; struct SMemData { diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 3400f8c516..7c877b463a 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -103,7 +103,8 @@ Dwarf_Debug tDbg; static TdThreadOnce traceThreadInit = PTHREAD_ONCE_INIT; void endTrace() { - if (traceThreadInit != PTHREAD_ONCE_INIT) { + TdThreadOnce tmp = PTHREAD_ONCE_INIT; + if (memcmp(&traceThreadInit, &tmp, sizeof(TdThreadOnce)) != 0) { delete_lookup_table(&lookup_table); dwarf_finish(tDbg); } diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 9dcbafca7a..10a5475555 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -911,7 +911,8 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1) void taosStopCacheRefreshWorker(void) { stopRefreshWorker = true; - if(cacheThreadInit != PTHREAD_ONCE_INIT) taosThreadJoin(cacheRefreshWorker, NULL); + TdThreadOnce tmp = PTHREAD_ONCE_INIT; + if (memcmp(&cacheRefreshWorker, &tmp, sizeof(TdThreadOnce)) != 0) taosThreadJoin(cacheRefreshWorker, NULL); taosArrayDestroy(pCacheArrayList); } From a4944d55163b2436ba42b6b05fe1e35bf20ca490 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 11 May 2022 18:25:37 +0800 Subject: [PATCH 35/94] fix: calculate col offset --- source/common/src/trow.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 44bcd72a33..d1516403c1 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -1159,7 +1159,7 @@ bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, #ifdef TD_SUPPORT_BITMAP int16_t colIdx = -1; - if (pKvIdx) colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pKvIdx) / sizeof(SKvRowIdx); + if (pKvIdx) colIdx = POINTER_DISTANCE(pKvIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx); if (tdGetBitmapValType(pIter->pBitmap, colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { pVal->valType = TD_VTYPE_NONE; } @@ -1226,7 +1226,7 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell compareKvRowColId, TD_EQ); #ifdef TD_SUPPORT_BITMAP if (pIdx) { - colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pIdx) / sizeof(SKvRowIdx); + colIdx = POINTER_DISTANCE(pIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx); } #endif tdGetKvRowValOfCol(pVal, pRow, pIter->pBitmap, pIdx ? pIdx->offset : -1, colIdx); From 327b5cdae9d92a6317af87a52567156382d2fbbc Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 11 May 2022 19:02:39 +0800 Subject: [PATCH 36/94] stmt auto create table --- include/common/taosdef.h | 2 +- source/libs/scalar/src/filter.c | 2 +- source/libs/scalar/src/scalar.c | 7 ++--- source/libs/scalar/src/sclvector.c | 6 ++--- tests/script/api/batchprepare.c | 41 ++++++++++++++++++------------ 5 files changed, 34 insertions(+), 24 deletions(-) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 5384082da3..e1f8832edf 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -89,7 +89,7 @@ extern char *qtypeStr[]; #define TSDB_PORT_HTTP 11 -#define TD_DEBUG_PRINT_ROW +#undef TD_DEBUG_PRINT_ROW #ifdef __cplusplus } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index eaab8e1f53..fe8aa77693 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -260,7 +260,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { comparFn = 20; } else if (optr == OP_TYPE_LIKE) { comparFn = 9; - } else if (optr == OP_TYPE_LIKE) { + } else if (optr == OP_TYPE_NOT_LIKE) { comparFn = 27; } else if (optr == OP_TYPE_IN) { comparFn = 8; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 5231890821..28aaafba0a 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -92,8 +92,9 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { } if (IS_VAR_DATA_TYPE(type)) { - len = varDataLen(out.columnData->pData); - buf = varDataVal(out.columnData->pData); + char* data = colDataGetVarData(out.columnData, 0); + len = varDataLen(data); + buf = varDataVal(data); } else { len = tDataTypes[type].bytes; buf = out.columnData->pData; @@ -109,7 +110,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { } if (taosHashPut(pObj, buf, (size_t)len, NULL, 0)) { - sclError("taosHashPut failed"); + sclError("taosHashPut to set failed"); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index b83147bfee..007f26ebc2 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -512,7 +512,7 @@ int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int1 if (outType == TSDB_DATA_TYPE_NCHAR) { varToNchar(tmp, pOut, i); } else { - colDataAppend(pOutputCol, i, (char *)&value, false); + colDataAppend(pOutputCol, i, (char *)tmp, false); } } } else if (IS_UNSIGNED_NUMERIC_TYPE(inType)) { @@ -529,7 +529,7 @@ int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int1 if (outType == TSDB_DATA_TYPE_NCHAR) { varToNchar(tmp, pOut, i); } else { - colDataAppend(pOutputCol, i, (char *)&value, false); + colDataAppend(pOutputCol, i, (char *)tmp, false); } } } else if (IS_FLOAT_TYPE(inType)) { @@ -546,7 +546,7 @@ int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int1 if (outType == TSDB_DATA_TYPE_NCHAR) { varToNchar(tmp, pOut, i); } else { - colDataAppend(pOutputCol, i, (char *)&value, false); + colDataAppend(pOutputCol, i, (char *)tmp, false); } } } else { diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 9f9ed88b89..bf38412b75 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -11,8 +11,8 @@ int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; -int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_NCHAR}; -int32_t optrIdxList[] = {0, 1, 2}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_NCHAR}; +int32_t optrIdxList[] = {0, 9}; typedef struct { char* oper; @@ -184,7 +184,7 @@ typedef struct { int32_t caseRunNum; // total run case num } CaseCtrl; -#if 0 +#if 1 CaseCtrl gCaseCtrl = { // default .bindNullNum = 0, .printCreateTblSql = false, @@ -202,7 +202,7 @@ CaseCtrl gCaseCtrl = { // default .optrIdxListNum = 0, .optrIdxList = NULL, .checkParamNum = false, - .printRes = true, + .printRes = false, .runTimes = 0, .caseIdx = -1, .caseNum = -1, @@ -212,7 +212,7 @@ CaseCtrl gCaseCtrl = { // default #endif -#if 1 +#if 0 CaseCtrl gCaseCtrl = { .bindNullNum = 0, .printCreateTblSql = true, @@ -223,18 +223,18 @@ CaseCtrl gCaseCtrl = { .bindColNum = 0, .bindTagNum = 0, .bindRowNum = 0, - .bindColTypeNum = 0, - .bindColTypeList = NULL, + .bindColTypeNum = tListLen(bindColTypeList), + .bindColTypeList = bindColTypeList, .bindTagTypeNum = 0, .bindTagTypeList = NULL, - .optrIdxListNum = 0, - .optrIdxList = NULL, + .optrIdxListNum = tListLen(optrIdxList), + .optrIdxList = optrIdxList, .checkParamNum = false, .printRes = false, .runTimes = 0, - .caseIdx = -1, + .caseIdx = 23, .caseNum = 1, - .caseRunIdx = 20, + .caseRunIdx = -1, .caseRunNum = 1, }; #endif @@ -253,14 +253,14 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper .optrIdxListNum = 0, .optrIdxList = NULL, .checkParamNum = false, - .printRes = true, + .printRes = false, .runTimes = 0, .caseRunIdx = -1, .optrIdxListNum = 0, .optrIdxList = NULL, .bindColTypeNum = 0, .bindColTypeList = NULL, - .caseIdx = 22, + .caseIdx = 23, .caseNum = 1, .caseRunNum = 1, }; @@ -3332,6 +3332,7 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) { TAOS_STMT *stmt = NULL; int64_t beginUs, endUs, totalUs; CaseCfg cfg = gCase[caseIdx]; + CaseCfg cfgBk; gCurCase = &cfg; if ((gCaseCtrl.bindColTypeNum || gCaseCtrl.bindColNum) && (gCurCase->colNum != gFullColNum)) { @@ -3402,6 +3403,7 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) { } totalUs = 0; + cfgBk = cfg; for (int32_t n = 0; n < gCurCase->runTimes; ++n) { if (gCurCase->preCaseIdx < 0) { prepare(taos, gCurCase->colNum, gCurCase->colList, gCurCase->autoCreateTbl); @@ -3423,6 +3425,8 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) { totalUs += (endUs - beginUs); prepareCheckResult(taos, silent); + + cfg = cfgBk; } if (!silent) { @@ -3465,18 +3469,19 @@ void* runCaseList(TAOS *taos) { } void runAll(TAOS *taos) { -#if 0 +#if 1 + strcpy(gCaseCtrl.caseCatalog, "Normal Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); runCaseList(taos); -#endif -#if 1 + strcpy(gCaseCtrl.caseCatalog, "Auto Create Table Test"); gCaseCtrl.autoCreateTbl = true; printf("%s Begin\n", gCaseCtrl.caseCatalog); runCaseList(taos); gCaseCtrl.autoCreateTbl = false; + #endif /* @@ -3499,6 +3504,7 @@ void runAll(TAOS *taos) { runCaseList(taos); gCaseCtrl.rowNum = 0; gCaseCtrl.printRes = true; +*/ strcpy(gCaseCtrl.caseCatalog, "Runtimes Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); @@ -3506,12 +3512,15 @@ void runAll(TAOS *taos) { runCaseList(taos); gCaseCtrl.runTimes = 0; +#if 1 strcpy(gCaseCtrl.caseCatalog, "Check Param Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.checkParamNum = true; runCaseList(taos); gCaseCtrl.checkParamNum = false; +#endif +/* strcpy(gCaseCtrl.caseCatalog, "Bind Col Num Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.bindColNum = 6; From 41e36dddb786a26b336162057e8877519a56881b Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 11 May 2022 19:03:59 +0800 Subject: [PATCH 37/94] fix: some problems of parser --- include/common/tvariant.h | 9 +- include/libs/planner/planner.h | 3 +- source/client/src/clientStmt.c | 220 +++++++-------- source/common/src/tvariant.c | 114 ++------ source/common/test/commonTests.cpp | 91 +++---- source/libs/function/src/builtins.c | 42 +-- source/libs/parser/src/parAstCreater.c | 21 +- source/libs/parser/src/parInsert.c | 155 ++++++----- source/libs/parser/src/parTranslater.c | 3 + source/libs/parser/test/parInitialCTest.cpp | 279 +++++++++++++++----- source/libs/planner/src/planner.c | 141 +++++++++- 11 files changed, 639 insertions(+), 439 deletions(-) diff --git a/include/common/tvariant.h b/include/common/tvariant.h index 83dccd0092..9728e5ecd5 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -36,12 +36,11 @@ typedef struct SVariant { }; } SVariant; -int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value, bool *issigned); +int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value); +int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value); bool taosVariantIsValid(SVariant *pVar); -void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type); - void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uint32_t type); void taosVariantDestroy(SVariant *pV); @@ -59,10 +58,10 @@ int32_t taosVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool #endif int32_t taosVariantTypeSetType(SVariant *pVariant, char type); -char * taosVariantGet(SVariant *pVar, int32_t type); +char *taosVariantGet(SVariant *pVar, int32_t type); #ifdef __cplusplus } #endif -#endif /*_TD_COMMON_VARIANT_H_*/ +#endif /*_TD_COMMON_VARIANT_H_*/ diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 367407cae5..e250b7b2b2 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -48,7 +48,8 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo // @pSource one execution location of this group of datasource subplans int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource); -int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId); +int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId, + bool* pEmptyResult); // Convert to subplan to string for the scheduler to send to the executor int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen); diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index c804213d89..d69ab4413b 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -1,12 +1,13 @@ #include "clientInt.h" #include "clientLog.h" -#include "clientStmt.h" #include "tdef.h" +#include "clientStmt.h" + int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { int32_t code = 0; - + switch (newStatus) { case STMT_PREPARE: break; @@ -29,11 +30,11 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND_COL)) { code = TSDB_CODE_TSC_STMT_API_ERROR; } -/* - if ((pStmt->sql.type == STMT_TYPE_MULTI_INSERT) && ()) { - code = TSDB_CODE_TSC_STMT_API_ERROR; - } -*/ + /* + if ((pStmt->sql.type == STMT_TYPE_MULTI_INSERT) && ()) { + code = TSDB_CODE_TSC_STMT_API_ERROR; + } + */ break; case STMT_BIND_COL: if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND)) { @@ -62,12 +63,11 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { return TSDB_CODE_SUCCESS; } - -int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) { +int32_t stmtGetTbName(TAOS_STMT* stmt, char** tbName) { STscStmt* pStmt = (STscStmt*)stmt; pStmt->sql.type = STMT_TYPE_MULTI_INSERT; - + if (NULL == pStmt->bInfo.tbName) { tscError("no table name set"); STMT_ERR_RET(TSDB_CODE_TSC_STMT_TBNAME_ERROR); @@ -79,10 +79,10 @@ int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) { } int32_t stmtBackupQueryFields(STscStmt* pStmt) { - SStmtQueryResInfo *pRes = &pStmt->sql.queryRes; + SStmtQueryResInfo* pRes = &pStmt->sql.queryRes; pRes->numOfCols = pStmt->exec.pRequest->body.resInfo.numOfCols; pRes->precision = pStmt->exec.pRequest->body.resInfo.precision; - + int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD); pRes->fields = taosMemoryMalloc(size); pRes->userFields = taosMemoryMalloc(size); @@ -96,9 +96,9 @@ int32_t stmtBackupQueryFields(STscStmt* pStmt) { } int32_t stmtRestoreQueryFields(STscStmt* pStmt) { - SStmtQueryResInfo *pRes = &pStmt->sql.queryRes; - int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD); - + SStmtQueryResInfo* pRes = &pStmt->sql.queryRes; + int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD); + pStmt->exec.pRequest->body.resInfo.numOfCols = pRes->numOfCols; pStmt->exec.pRequest->body.resInfo.precision = pRes->precision; @@ -151,12 +151,12 @@ int32_t stmtGetExecInfo(TAOS_STMT* stmt, SHashObj** pVgHash, SHashObj** pBlockHa return TSDB_CODE_SUCCESS; } -int32_t stmtCacheBlock(STscStmt *pStmt) { +int32_t stmtCacheBlock(STscStmt* pStmt) { if (pStmt->sql.type != STMT_TYPE_MULTI_INSERT) { return TSDB_CODE_SUCCESS; } - uint64_t uid = pStmt->bInfo.tbUid; + uint64_t uid = pStmt->bInfo.tbUid; uint64_t tuid = (TSDB_CHILD_TABLE == pStmt->bInfo.tbType) ? pStmt->bInfo.tbSuid : uid; if (taosHashGet(pStmt->sql.pTableCache, &tuid, sizeof(tuid))) { @@ -164,13 +164,13 @@ int32_t stmtCacheBlock(STscStmt *pStmt) { } STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, &uid, sizeof(uid)); - STableDataBlocks* pDst = NULL; - + STableDataBlocks* pDst = NULL; + STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc)); SStmtTableCache cache = { - .pDataBlock = pDst, - .boundTags = pStmt->bInfo.boundTags, + .pDataBlock = pDst, + .boundTags = pStmt->bInfo.boundTags, }; if (taosHashPut(pStmt->sql.pTableCache, &tuid, sizeof(tuid), &cache, sizeof(cache))) { @@ -184,21 +184,21 @@ int32_t stmtCacheBlock(STscStmt *pStmt) { int32_t stmtParseSql(STscStmt* pStmt) { SStmtCallback stmtCb = { - .pStmt = pStmt, - .getTbNameFn = stmtGetTbName, - .setBindInfoFn = stmtSetBindInfo, - .setExecInfoFn = stmtSetExecInfo, - .getExecInfoFn = stmtGetExecInfo, + .pStmt = pStmt, + .getTbNameFn = stmtGetTbName, + .setBindInfoFn = stmtSetBindInfo, + .setExecInfoFn = stmtSetExecInfo, + .getExecInfoFn = stmtGetExecInfo, }; if (NULL == pStmt->exec.pRequest) { STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); } - + STMT_ERR_RET(parseSql(pStmt->exec.pRequest, false, &pStmt->sql.pQuery, &stmtCb)); pStmt->bInfo.needParse = false; - + switch (nodeType(pStmt->sql.pQuery->pRoot)) { case QUERY_NODE_VNODE_MODIF_STMT: if (0 == pStmt->sql.type) { @@ -237,14 +237,14 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) { pStmt->exec.pRequest = NULL; } - void *pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); + void* pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { - STableDataBlocks* pBlocks = *(STableDataBlocks**)pIter; - uint64_t *key = taosHashGetKey(pIter, NULL); - + STableDataBlocks* pBlocks = *(STableDataBlocks**)pIter; + uint64_t* key = taosHashGetKey(pIter, NULL); + if (keepTable && (*key == pStmt->bInfo.tbUid)) { STMT_ERR_RET(qResetStmtDataBlock(pBlocks, true)); - + pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); continue; } @@ -274,15 +274,15 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { qDestroyQuery(pStmt->sql.pQuery); qDestroyQueryPlan(pStmt->sql.pQueryPlan); taosArrayDestroy(pStmt->sql.nodeList); - - void *pIter = taosHashIterate(pStmt->sql.pTableCache, NULL); + + void* pIter = taosHashIterate(pStmt->sql.pTableCache, NULL); while (pIter) { - SStmtTableCache* pCache = (SStmtTableCache*)pIter; + SStmtTableCache* pCache = (SStmtTableCache*)pIter; qDestroyStmtDataBlock(pCache->pDataBlock); destroyBoundColumnInfo(pCache->boundTags); taosMemoryFreeClear(pCache->boundTags); - + pIter = taosHashIterate(pStmt->sql.pTableCache, pIter); } taosHashCleanup(pStmt->sql.pTableCache); @@ -307,22 +307,23 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { STMT_ERR_RET(catalogGetHandle(pStmt->taos->pAppInfo->clusterId, &pStmt->pCatalog)); } - STableMeta *pTableMeta = NULL; - SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); - int32_t code = catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta); + STableMeta* pTableMeta = NULL; + SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); + int32_t code = + catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta); if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { STMT_ERR_RET(stmtCleanBindInfo(pStmt)); - + return TSDB_CODE_SUCCESS; } STMT_ERR_RET(code); - + uint64_t uid = pTableMeta->uid; uint64_t suid = pTableMeta->suid; - int8_t tableType = pTableMeta->tableType; + int8_t tableType = pTableMeta->tableType; taosMemoryFree(pTableMeta); - + if (uid == pStmt->bInfo.tbUid) { pStmt->bInfo.needParse = false; @@ -333,12 +334,12 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid)); if (NULL == pCache) { tscError("table uid %" PRIx64 "found in exec blockHash, but not in sql blockHash", uid); - + STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); } - + pStmt->bInfo.needParse = false; - + pStmt->bInfo.tbUid = uid; pStmt->bInfo.tbSuid = suid; pStmt->bInfo.tbType = tableType; @@ -361,10 +362,11 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { STableDataBlocks* pNewBlock = NULL; STMT_ERR_RET(qRebuildStmtDataBlock(&pNewBlock, pCache->pDataBlock)); - if (taosHashPut(pStmt->exec.pBlockHash, &pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid), &pNewBlock, POINTER_BYTES)) { + if (taosHashPut(pStmt->exec.pBlockHash, &pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid), &pNewBlock, + POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + return TSDB_CODE_SUCCESS; } @@ -387,9 +389,8 @@ int32_t stmtResetStmt(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } - -TAOS_STMT *stmtInit(TAOS *taos) { - STscObj* pObj = (STscObj*)taos; +TAOS_STMT* stmtInit(TAOS* taos) { + STscObj* pObj = (STscObj*)taos; STscStmt* pStmt = NULL; pStmt = taosMemoryCalloc(1, sizeof(STscStmt)); @@ -408,11 +409,11 @@ TAOS_STMT *stmtInit(TAOS *taos) { pStmt->taos = pObj; pStmt->bInfo.needParse = true; pStmt->sql.status = STMT_INIT; - + return pStmt; } -int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { +int stmtPrepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { STscStmt* pStmt = (STscStmt*)stmt; if (pStmt->sql.status >= STMT_PREPARE) { @@ -424,15 +425,14 @@ int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { if (length <= 0) { length = strlen(sql); } - + pStmt->sql.sqlStr = strndup(sql, length); pStmt->sql.sqlLen = length; return TSDB_CODE_SUCCESS; } - -int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) { +int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) { STscStmt* pStmt = (STscStmt*)stmt; STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTBNAME)); @@ -447,9 +447,10 @@ int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) { if (NULL == pStmt->exec.pRequest) { STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); } - - STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); - + + STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, + pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); + STMT_ERR_RET(stmtGetFromCache(pStmt)); if (pStmt->bInfo.needParse) { @@ -460,7 +461,7 @@ int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) { return TSDB_CODE_SUCCESS; } -int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags) { +int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { STscStmt* pStmt = (STscStmt*)stmt; STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS)); @@ -471,25 +472,27 @@ int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags) { STMT_ERR_RET(stmtParseSql(pStmt)); - STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); + STableDataBlocks** pDataBlock = (STableDataBlocks**)taosHashGet( + pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); if (NULL == pDataBlock) { tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - - STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); + + STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.sname.tname, + tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); return TSDB_CODE_SUCCESS; } - -int32_t stmtFetchTagFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fields) { +int32_t stmtFetchTagFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD** fields) { if (STMT_TYPE_QUERY == pStmt->sql.type) { tscError("invalid operation to get query tag fileds"); STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); + STableDataBlocks** pDataBlock = (STableDataBlocks**)taosHashGet( + pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); if (NULL == pDataBlock) { tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -500,13 +503,14 @@ int32_t stmtFetchTagFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fiel return TSDB_CODE_SUCCESS; } -int32_t stmtFetchColFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fields) { +int32_t stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD** fields) { if (STMT_TYPE_QUERY == pStmt->sql.type) { tscError("invalid operation to get query column fileds"); STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); + STableDataBlocks** pDataBlock = (STableDataBlocks**)taosHashGet( + pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); if (NULL == pDataBlock) { tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -514,13 +518,14 @@ int32_t stmtFetchColFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fiel STMT_ERR_RET(qBuildStmtColFields(*pDataBlock, fieldNum, fields)); - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } -int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) { +int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { STscStmt* pStmt = (STscStmt*)stmt; - if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { + if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && + STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { pStmt->bInfo.needParse = false; } @@ -528,7 +533,7 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) { taos_free_result(pStmt->exec.pRequest); pStmt->exec.pRequest = NULL; } - + if (NULL == pStmt->exec.pRequest) { STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); } @@ -548,11 +553,13 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) { } else { STMT_ERR_RET(stmtRestoreQueryFields(pStmt)); } - - STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx, pStmt->exec.pRequest->requestId)); + + bool emptyResult = false; + STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx, pStmt->exec.pRequest->requestId, &emptyResult)); } - - STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); + + STableDataBlocks** pDataBlock = (STableDataBlocks**)taosHashGet( + pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); if (NULL == pDataBlock) { tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -571,31 +578,31 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) { } pStmt->bInfo.sBindLastIdx = colIdx; - + if (0 == colIdx) { pStmt->bInfo.sBindRowNum = bind->num; } - - qBindStmtSingleColValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum); + + qBindStmtSingleColValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, colIdx, + pStmt->bInfo.sBindRowNum); } - + return TSDB_CODE_SUCCESS; } - -int stmtAddBatch(TAOS_STMT *stmt) { +int stmtAddBatch(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_ADD_BATCH)); STMT_ERR_RET(stmtCacheBlock(pStmt)); - + return TSDB_CODE_SUCCESS; } -int stmtExec(TAOS_STMT *stmt) { +int stmtExec(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; - int32_t code = 0; + int32_t code = 0; STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); @@ -615,7 +622,7 @@ int stmtExec(TAOS_STMT *stmt) { STMT_ERR_RET(TSDB_CODE_NEED_RETRY); } } - + STMT_ERR_JRET(pStmt->exec.pRequest->code); pStmt->exec.affectedRows = taos_affected_rows(pStmt->exec.pRequest); @@ -624,14 +631,13 @@ int stmtExec(TAOS_STMT *stmt) { _return: stmtCleanExecInfo(pStmt, (code ? false : true), false); - + ++pStmt->sql.runTimes; - + STMT_RET(code); } - -int stmtClose(TAOS_STMT *stmt) { +int stmtClose(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; STMT_RET(stmtCleanSQLInfo(pStmt)); @@ -639,11 +645,11 @@ int stmtClose(TAOS_STMT *stmt) { taosMemoryFree(stmt); } -const char *stmtErrstr(TAOS_STMT *stmt) { +const char* stmtErrstr(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; if (stmt == NULL || NULL == pStmt->exec.pRequest) { - return (char*) tstrerror(terrno); + return (char*)tstrerror(terrno); } pStmt->exec.pRequest->code = terrno; @@ -651,15 +657,11 @@ const char *stmtErrstr(TAOS_STMT *stmt) { return taos_errstr(pStmt->exec.pRequest); } -int stmtAffectedRows(TAOS_STMT *stmt) { - return ((STscStmt*)stmt)->affectedRows; -} +int stmtAffectedRows(TAOS_STMT* stmt) { return ((STscStmt*)stmt)->affectedRows; } -int stmtAffectedRowsOnce(TAOS_STMT *stmt) { - return ((STscStmt*)stmt)->exec.affectedRows; -} +int stmtAffectedRowsOnce(TAOS_STMT* stmt) { return ((STscStmt*)stmt)->exec.affectedRows; } -int stmtIsInsert(TAOS_STMT *stmt, int *insert) { +int stmtIsInsert(TAOS_STMT* stmt, int* insert) { STscStmt* pStmt = (STscStmt*)stmt; if (pStmt->sql.type) { @@ -667,16 +669,17 @@ int stmtIsInsert(TAOS_STMT *stmt, int *insert) { } else { *insert = isInsertSql(pStmt->sql.sqlStr, 0); } - + return TSDB_CODE_SUCCESS; } -int stmtGetParamNum(TAOS_STMT *stmt, int *nums) { +int stmtGetParamNum(TAOS_STMT* stmt, int* nums) { STscStmt* pStmt = (STscStmt*)stmt; STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); - if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { + if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && + STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { pStmt->bInfo.needParse = false; } @@ -684,7 +687,7 @@ int stmtGetParamNum(TAOS_STMT *stmt, int *nums) { taos_free_result(pStmt->exec.pRequest); pStmt->exec.pRequest = NULL; } - + if (NULL == pStmt->exec.pRequest) { STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); } @@ -702,16 +705,16 @@ int stmtGetParamNum(TAOS_STMT *stmt, int *nums) { } else { STMT_ERR_RET(stmtRestoreQueryFields(pStmt)); } - + *nums = taosArrayGetSize(pStmt->sql.pQueryPlan->pPlaceholderValues); } else { STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL)); } - + return TSDB_CODE_SUCCESS; } -TAOS_RES *stmtUseResult(TAOS_STMT *stmt) { +TAOS_RES* stmtUseResult(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; if (STMT_TYPE_QUERY != pStmt->sql.type) { @@ -721,6 +724,3 @@ TAOS_RES *stmtUseResult(TAOS_STMT *stmt) { return pStmt->exec.pRequest; } - - - diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index a546d9bb6b..e44450fe6e 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -35,104 +35,36 @@ assert(0); \ } while (0) -int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value, bool *isSigned) { +int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) { errno = 0; char *endPtr = NULL; - int32_t index = 0; - - bool specifiedSign = (z[0] == '+' || z[0] == '-'); - if (specifiedSign) { - *isSigned = true; - index = 1; - } - - uint64_t val = strtoull(&z[index], &endPtr, base); - if (errno == ERANGE || errno == EINVAL) { + *value = strtoll(z, &endPtr, base); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { errno = 0; return -1; } - if (specifiedSign && val > INT64_MAX) { - return -1; - } - - if (endPtr - &z[index] != n - index) { - return -1; - } - - *isSigned = specifiedSign || (val <= INT64_MAX); - if (*isSigned) { - *value = (z[0] == '-') ? -val : val; - } else { - *(uint64_t *)value = val; - } - return 0; } -void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type) { - int32_t ret = 0; - memset(pVar, 0, sizeof(SVariant)); +int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) { + errno = 0; + char *endPtr = NULL; - switch (type) { - case TSDB_DATA_TYPE_BOOL: { - if (strncasecmp(z, "true", 4) == 0) { - pVar->i = TSDB_TRUE; - } else if (strncasecmp(z, "false", 5) == 0) { - pVar->i = TSDB_FALSE; - } else { - return; - } - break; - } - - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_INT: { - bool sign = true; - - int32_t base = 10; - if (type == TK_NK_HEX) { - base = 16; - } else if (type == TK_NK_OCT) { - base = 8; - } else if (type == TK_NK_BIN) { - base = 2; - } - - ret = toInteger(z, n, base, &pVar->i, &sign); - if (ret != 0) { - pVar->nType = -1; // -1 means error type - return; - } - - pVar->nType = (sign) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_UBIGINT; - break; - } - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_FLOAT: { - pVar->d = strtod(z, NULL); - break; - } - case TSDB_DATA_TYPE_BINARY: { - pVar->pz = strndup(z, n); - //pVar->nLen = strRmquote(pVar->pz, n); - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: { - assert(0); - pVar->i = taosGetTimestamp(TSDB_TIME_PRECISION_NANO); - break; - } - - default: { // nType == 0 means the null value - type = TSDB_DATA_TYPE_NULL; - } + const char *p = z; + while (*p != 0 && *p == ' ') p++; + if (*p != 0 && *p == '-') { + return -1; } - pVar->nType = type; + *value = strtoull(z, &endPtr, base); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + errno = 0; + return -1; + } + + return 0; } /** @@ -461,7 +393,7 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) { if (*pDest == pVariant->pz) { TdUcs4 *pWStr = taosMemoryCalloc(1, (nLen + 1) * TSDB_NCHAR_SIZE); - bool ret = taosMbsToUcs4(pDst, nLen, pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); + bool ret = taosMbsToUcs4(pDst, nLen, pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); if (!ret) { taosMemoryFreeClear(pWStr); return -1; @@ -483,7 +415,7 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) { } else { int32_t output = 0; - bool ret = taosMbsToUcs4(pDst, nLen, (TdUcs4*)*pDest, (nLen + 1) * TSDB_NCHAR_SIZE, &output); + bool ret = taosMbsToUcs4(pDst, nLen, (TdUcs4 *)*pDest, (nLen + 1) * TSDB_NCHAR_SIZE, &output); if (!ret) { return -1; } @@ -518,9 +450,9 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result } else if (IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { *result = pVariant->u; } else if (IS_FLOAT_TYPE(pVariant->nType)) { - *result = (int64_t) pVariant->d; + *result = (int64_t)pVariant->d; } else { - //TODO: handling var types + // TODO: handling var types } #if 0 errno = 0; @@ -909,7 +841,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc return -1; } } else { - tasoUcs4Copy((TdUcs4*)payload, pVariant->ucs4, pVariant->nLen); + tasoUcs4Copy((TdUcs4 *)payload, pVariant->ucs4, pVariant->nLen); } } } else { @@ -1026,7 +958,7 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { return 0; } -char * taosVariantGet(SVariant *pVar, int32_t type) { +char *taosVariantGet(SVariant *pVar, int32_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index fea6c4c891..2adb558d01 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -8,12 +8,11 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" +#include "taos.h" #include "tcommon.h" #include "tdatablock.h" -#include "tcommon.h" -#include "taos.h" -#include "tvariant.h" #include "tdef.h" +#include "tvariant.h" namespace { // @@ -29,72 +28,62 @@ TEST(testCase, toInteger_test) { uint32_t type = 0; int64_t val = 0; - bool sign = true; - int32_t ret = toInteger(s, strlen(s), 10, &val, &sign); + int32_t ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 123); - ASSERT_EQ(sign, true); s = "9223372036854775807"; - ret = toInteger(s, strlen(s), 10, &val, &sign); + ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 9223372036854775807); - ASSERT_EQ(sign, true); s = "9323372036854775807"; - ret = toInteger(s, strlen(s), 10, &val, &sign); + ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 9323372036854775807u); - ASSERT_EQ(sign, false); s = "-9323372036854775807"; - ret = toInteger(s, strlen(s), 10, &val, &sign); + ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, -1); s = "-1"; - ret = toInteger(s, strlen(s), 10, &val, &sign); + ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -1); - ASSERT_EQ(sign, true); s = "-9223372036854775807"; - ret = toInteger(s, strlen(s), 10, &val, &sign); + ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -9223372036854775807); - ASSERT_EQ(sign, true); s = "1000u"; - ret = toInteger(s, strlen(s), 10, &val, &sign); + ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, -1); s = "0x10"; - ret = toInteger(s, strlen(s), 16, &val, &sign); + ret = toInteger(s, strlen(s), 16, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 16); - ASSERT_EQ(sign, true); s = "110"; - ret = toInteger(s, strlen(s), 2, &val, &sign); + ret = toInteger(s, strlen(s), 2, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 6); - ASSERT_EQ(sign, true); s = "110"; - ret = toInteger(s, strlen(s), 8, &val, &sign); + ret = toInteger(s, strlen(s), 8, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 72); - ASSERT_EQ(sign, true); - //18446744073709551615 UINT64_MAX + // 18446744073709551615 UINT64_MAX s = "18446744073709551615"; - ret = toInteger(s, strlen(s), 10, &val, &sign); + ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 18446744073709551615u); - ASSERT_EQ(sign, false); s = "18446744073709551616"; - ret = toInteger(s, strlen(s), 10, &val, &sign); + ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, -1); } @@ -108,8 +97,8 @@ TEST(testCase, Datablock_test) { infoData.info.type = TSDB_DATA_TYPE_INT; infoData.info.colId = 1; - infoData.pData = (char*) taosMemoryCalloc(40, infoData.info.bytes); - infoData.nullbitmap = (char*) taosMemoryCalloc(1, sizeof(char) * (40/8)); + infoData.pData = (char*)taosMemoryCalloc(40, infoData.info.bytes); + infoData.nullbitmap = (char*)taosMemoryCalloc(1, sizeof(char) * (40 / 8)); taosArrayPush(b->pDataBlock, &infoData); SColumnInfoData infoData1 = {0}; @@ -117,36 +106,36 @@ TEST(testCase, Datablock_test) { infoData1.info.type = TSDB_DATA_TYPE_BINARY; infoData1.info.colId = 2; - infoData1.varmeta.offset = (int32_t*) taosMemoryCalloc(40, sizeof(uint32_t)); + infoData1.varmeta.offset = (int32_t*)taosMemoryCalloc(40, sizeof(uint32_t)); taosArrayPush(b->pDataBlock, &infoData1); char* str = "the value of: %d"; - char buf[128] = {0}; - char varbuf[128] = {0}; + char buf[128] = {0}; + char varbuf[128] = {0}; - for(int32_t i = 0; i < 40; ++i) { - SColumnInfoData* p0 = (SColumnInfoData *) taosArrayGet(b->pDataBlock, 0); - SColumnInfoData* p1 = (SColumnInfoData *) taosArrayGet(b->pDataBlock, 1); + for (int32_t i = 0; i < 40; ++i) { + SColumnInfoData* p0 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 0); + SColumnInfoData* p1 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 1); - if (i&0x01) { + if (i & 0x01) { int32_t len = sprintf(buf, str, i); STR_TO_VARSTR(varbuf, buf) - colDataAppend(p0, i, (const char*) &i, false); - colDataAppend(p1, i, (const char*) varbuf, false); + colDataAppend(p0, i, (const char*)&i, false); + colDataAppend(p1, i, (const char*)varbuf, false); memset(varbuf, 0, sizeof(varbuf)); memset(buf, 0, sizeof(buf)); } else { - colDataAppend(p0, i, (const char*) &i, true); - colDataAppend(p1, i, (const char*) varbuf, true); + colDataAppend(p0, i, (const char*)&i, true); + colDataAppend(p1, i, (const char*)varbuf, true); } b->info.rows++; } - SColumnInfoData* p0 = (SColumnInfoData *) taosArrayGet(b->pDataBlock, 0); - SColumnInfoData* p1 = (SColumnInfoData *) taosArrayGet(b->pDataBlock, 1); - for(int32_t i = 0; i < 40; ++i) { + SColumnInfoData* p0 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 0); + SColumnInfoData* p1 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 1); + for (int32_t i = 0; i < 40; ++i) { if (i & 0x01) { ASSERT_EQ(colDataIsNull_f(p0->nullbitmap, i), false); ASSERT_EQ(colDataIsNull(p1, b->info.rows, i, nullptr), false); @@ -158,7 +147,7 @@ TEST(testCase, Datablock_test) { } } - printf("binary column length:%d\n", *(int32_t*) p1->pData); + printf("binary column length:%d\n", *(int32_t*)p1->pData); ASSERT_EQ(blockDataGetNumOfCols(b), 2); ASSERT_EQ(blockDataGetNumOfRows(b), 40); @@ -166,8 +155,8 @@ TEST(testCase, Datablock_test) { char* pData = colDataGetData(p1, 3); printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData)); - SArray* pOrderInfo = taosArrayInit(3, sizeof(SBlockOrderInfo)); - SBlockOrderInfo order = { true, TSDB_ORDER_ASC, 0, NULL }; + SArray* pOrderInfo = taosArrayInit(3, sizeof(SBlockOrderInfo)); + SBlockOrderInfo order = {true, TSDB_ORDER_ASC, 0, NULL}; taosArrayPush(pOrderInfo, &order); blockDataSort(b, pOrderInfo); @@ -244,8 +233,8 @@ TEST(testCase, var_dataBlock_split_test) { infoData.info.type = TSDB_DATA_TYPE_INT; infoData.info.colId = 1; - infoData.pData = (char*) taosMemoryCalloc(numOfRows, infoData.info.bytes); - infoData.nullbitmap = (char*) taosMemoryCalloc(1, sizeof(char) * (numOfRows/8)); + infoData.pData = (char*)taosMemoryCalloc(numOfRows, infoData.info.bytes); + infoData.nullbitmap = (char*)taosMemoryCalloc(1, sizeof(char) * (numOfRows / 8)); taosArrayPush(b->pDataBlock, &infoData); SColumnInfoData infoData1 = {0}; @@ -253,13 +242,13 @@ TEST(testCase, var_dataBlock_split_test) { infoData1.info.type = TSDB_DATA_TYPE_BINARY; infoData1.info.colId = 2; - infoData1.varmeta.offset = (int32_t*) taosMemoryCalloc(numOfRows, sizeof(uint32_t)); + infoData1.varmeta.offset = (int32_t*)taosMemoryCalloc(numOfRows, sizeof(uint32_t)); taosArrayPush(b->pDataBlock, &infoData1); char buf[41] = {0}; char buf1[100] = {0}; - for(int32_t i = 0; i < numOfRows; ++i) { + for (int32_t i = 0; i < numOfRows; ++i) { SColumnInfoData* p0 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 0); SColumnInfoData* p1 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 1); @@ -278,10 +267,10 @@ TEST(testCase, var_dataBlock_split_test) { int32_t pageSize = 64 * 1024; - int32_t startIndex= 0; + int32_t startIndex = 0; int32_t stopIndex = 0; int32_t count = 1; - while(1) { + while (1) { blockDataSplitRows(b, true, startIndex, &stopIndex, pageSize); printf("the %d split, from: %d to %d\n", count++, startIndex, stopIndex); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 80bfb626d8..5e3259c8fb 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -239,7 +239,7 @@ static int32_t translateLeastSQR(SFunctionNode* pFunc, char* pErrBuf, int32_t le } } - pFunc->node.resType = (SDataType) { .bytes = 64, .type = TSDB_DATA_TYPE_BINARY }; + pFunc->node.resType = (SDataType){.bytes = 64, .type = TSDB_DATA_TYPE_BINARY}; return TSDB_CODE_SUCCESS; } @@ -977,26 +977,26 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = timezoneFunction, .finalizeFunc = NULL }, - { - .name = "_rowts", - .type = FUNCTION_TYPE_ROWTS, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC, - .translateFunc = translateTimePseudoColumn, - .getEnvFunc = getTimePseudoFuncEnv, - .initFunc = NULL, - .sprocessFunc = NULL, - .finalizeFunc = NULL - }, - { - .name = "_c0", - .type = FUNCTION_TYPE_ROWTS, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC, - .translateFunc = translateTimePseudoColumn, - .getEnvFunc = getTimePseudoFuncEnv, - .initFunc = NULL, - .sprocessFunc = NULL, - .finalizeFunc = NULL - }, + // { + // .name = "_rowts", + // .type = FUNCTION_TYPE_ROWTS, + // .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC, + // .translateFunc = translateTimePseudoColumn, + // .getEnvFunc = getTimePseudoFuncEnv, + // .initFunc = NULL, + // .sprocessFunc = NULL, + // .finalizeFunc = NULL + // }, + // { + // .name = "_c0", + // .type = FUNCTION_TYPE_ROWTS, + // .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC, + // .translateFunc = translateTimePseudoColumn, + // .getEnvFunc = getTimePseudoFuncEnv, + // .initFunc = NULL, + // .sprocessFunc = NULL, + // .finalizeFunc = NULL + // }, { .name = "tbname", .type = FUNCTION_TYPE_TBNAME, diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a9481593e0..b7a14a81c6 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -350,7 +350,18 @@ SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, createOperatorNode(pCxt, OP_TYPE_GREATER_THAN, nodesCloneNode(pExpr), pRight)); } +static SNode* createPrimaryKeyCol(SAstCreateContext* pCxt) { + SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); + CHECK_OUT_OF_MEM(pCol); + pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + strcpy(pCol->colName, PK_TS_COL_INTERNAL_NAME); + return (SNode*)pCol; +} + SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList) { + if (0 == strncasecmp("_rowts", pFuncName->z, pFuncName->n) || 0 == strncasecmp("_c0", pFuncName->z, pFuncName->n)) { + return createPrimaryKeyCol(pCxt); + } SFunctionNode* func = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); CHECK_OUT_OF_MEM(func); strncpy(func->functionName, pFuncName->z, pFuncName->n); @@ -467,13 +478,11 @@ SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr) { SStateWindowNode* state = (SStateWindowNode*)nodesMakeNode(QUERY_NODE_STATE_WINDOW); CHECK_OUT_OF_MEM(state); - state->pCol = nodesMakeNode(QUERY_NODE_COLUMN); + state->pCol = createPrimaryKeyCol(pCxt); if (NULL == state->pCol) { nodesDestroyNode(state); CHECK_OUT_OF_MEM(state->pCol); } - ((SColumnNode*)state->pCol)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - strcpy(((SColumnNode*)state->pCol)->colName, PK_TS_COL_INTERNAL_NAME); state->pExpr = pExpr; return (SNode*)state; } @@ -482,13 +491,11 @@ SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode SNode* pFill) { SIntervalWindowNode* interval = (SIntervalWindowNode*)nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); CHECK_OUT_OF_MEM(interval); - interval->pCol = nodesMakeNode(QUERY_NODE_COLUMN); + interval->pCol = createPrimaryKeyCol(pCxt); if (NULL == interval->pCol) { nodesDestroyNode(interval); CHECK_OUT_OF_MEM(interval->pCol); } - ((SColumnNode*)interval->pCol)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - strcpy(((SColumnNode*)interval->pCol)->colName, PK_TS_COL_INTERNAL_NAME); interval->pInterval = pInterval; interval->pOffset = pOffset; interval->pSliding = pSliding; @@ -667,7 +674,7 @@ SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOpti case DB_OPTION_DAYS: { SToken* pToken = pVal; if (TK_NK_INTEGER == pToken->type) { - ((SDatabaseOptions*)pOptions)->daysPerFile = strtol(pToken->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->daysPerFile = strtol(pToken->z, NULL, 10) * 1440; } else { ((SDatabaseOptions*)pOptions)->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, pToken); } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 64d2934282..87dc8b97c8 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -246,7 +246,8 @@ static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SToken* pTname, bool tNameGetFullDbName(&name, dbFname); bool pass = false; - CHECK_CODE(catalogChkAuth(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, pBasicCtx->pUser, dbFname, AUTH_TYPE_WRITE, &pass)); + CHECK_CODE(catalogChkAuth(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, pBasicCtx->pUser, + dbFname, AUTH_TYPE_WRITE, &pass)); if (!pass) { return TSDB_CODE_PAR_PERMISSION_DENIED; } @@ -349,8 +350,7 @@ static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time } else if (pToken->type == TK_TODAY) { ts = taosGetTimestampToday(timePrec); } else if (pToken->type == TK_NK_INTEGER) { - bool isSigned = false; - toInteger(pToken->z, pToken->n, 10, &ts, &isSigned); + toInteger(pToken->z, pToken->n, 10, &ts); } else { // parse the RFC-3339/ISO-8601 timestamp format string if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); @@ -453,9 +453,9 @@ static FORCE_INLINE int32_t toDouble(SToken* pToken, double* value, char** endPt static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, _row_append_fn_t func, void* param, SMsgBuf* pMsgBuf) { - int64_t iv; - char* endptr = NULL; - bool isSigned = false; + int64_t iv; + uint64_t uv; + char* endptr = NULL; int32_t code = checkAndTrimValue(pToken, pSchema->type, tmpTokenBuf, pMsgBuf); if (code != TSDB_CODE_SUCCESS) { @@ -490,7 +490,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int } case TSDB_DATA_TYPE_TINYINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z); } else if (!IS_VALID_TINYINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z); @@ -501,17 +501,17 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int } case TSDB_DATA_TYPE_UTINYINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); - } else if (!IS_VALID_UTINYINT(iv)) { + } else if (!IS_VALID_UTINYINT(uv)) { return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); } - uint8_t tmpVal = (uint8_t)iv; + uint8_t tmpVal = (uint8_t)uv; return func(pMsgBuf, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_SMALLINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z); } else if (!IS_VALID_SMALLINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z); @@ -521,17 +521,17 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int } case TSDB_DATA_TYPE_USMALLINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); - } else if (!IS_VALID_USMALLINT(iv)) { + } else if (!IS_VALID_USMALLINT(uv)) { return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); } - uint16_t tmpVal = (uint16_t)iv; + uint16_t tmpVal = (uint16_t)uv; return func(pMsgBuf, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_INT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z); } else if (!IS_VALID_INT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z); @@ -541,17 +541,17 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int } case TSDB_DATA_TYPE_UINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); - } else if (!IS_VALID_UINT(iv)) { + } else if (!IS_VALID_UINT(uv)) { return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); } - uint32_t tmpVal = (uint32_t)iv; + uint32_t tmpVal = (uint32_t)uv; return func(pMsgBuf, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_BIGINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); } else if (!IS_VALID_BIGINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z); @@ -560,13 +560,12 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int } case TSDB_DATA_TYPE_UBIGINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); - } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { + } else if (!IS_VALID_UBIGINT(uv)) { return buildSyntaxErrMsg(pMsgBuf, "unsigned bigint data overflow", pToken->z); } - uint64_t tmpVal = (uint64_t)iv; - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); + return func(pMsgBuf, &uv, pSchema->bytes, param); } case TSDB_DATA_TYPE_FLOAT: { @@ -771,7 +770,7 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, voi return TSDB_CODE_SUCCESS; } -static int32_t buildCreateTbReq(SVCreateTbReq *pTbReq, const char* tname, SKVRow row, int64_t suid) { +static int32_t buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, SKVRow row, int64_t suid) { pTbReq->type = TD_CHILD_TABLE; pTbReq->name = strdup(tname); pTbReq->ctb.suid = suid; @@ -1273,9 +1272,10 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash return TSDB_CODE_SUCCESS; } -int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, char *tName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen){ - STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; +int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tName, TAOS_MULTI_BIND* bind, + char* msgBuf, int32_t msgBufLen) { + STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; if (NULL == tags) { return TSDB_CODE_QRY_APP_ERROR; @@ -1550,16 +1550,16 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD** fields // schemaless logic start typedef struct SmlExecHandle { - SHashObj* pBlockHash; + SHashObj* pBlockHash; - SParsedDataColInfo tags; // each table - SKVRowBuilder tagsBuilder; // each table - SVCreateTbReq createTblReq; // each table + SParsedDataColInfo tags; // each table + SKVRowBuilder tagsBuilder; // each table + SVCreateTbReq createTblReq; // each table SQuery* pQuery; } SSmlExecHandle; -static int32_t smlBoundColumnData(SArray *cols, SParsedDataColInfo* pColList, SSchema* pSchema) { +static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema) { col_id_t nCols = pColList->numOfCols; pColList->numOfBound = 0; @@ -1572,8 +1572,8 @@ static int32_t smlBoundColumnData(SArray *cols, SParsedDataColInfo* pColList, SS bool isOrdered = true; col_id_t lastColIdx = -1; // last column found for (int i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv *kv = taosArrayGetP(cols, i); - SToken sToken = {.n=kv->keyLen, .z=(char*)kv->key}; + SSmlKv* kv = taosArrayGetP(cols, i); + SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key}; col_id_t t = lastColIdx + 1; col_id_t index = findCol(&sToken, t, nCols, pSchema); if (index < 0 && t > 0) { @@ -1622,7 +1622,7 @@ static int32_t smlBoundColumnData(SArray *cols, SParsedDataColInfo* pColList, SS qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); } - if(pColList->numOfCols > pColList->numOfBound){ + if (pColList->numOfCols > pColList->numOfBound) { memset(&pColList->boundColumns[pColList->numOfBound], 0, sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); } @@ -1630,53 +1630,52 @@ static int32_t smlBoundColumnData(SArray *cols, SParsedDataColInfo* pColList, SS return TSDB_CODE_SUCCESS; } -static int32_t smlBuildTagRow(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDataColInfo* tags, SSchema* pSchema, SKVRow *row, SMsgBuf *msg) { +static int32_t smlBuildTagRow(SArray* cols, SKVRowBuilder* tagsBuilder, SParsedDataColInfo* tags, SSchema* pSchema, + SKVRow* row, SMsgBuf* msg) { if (tdInitKVRowBuilder(tagsBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } SKvParam param = {.builder = tagsBuilder}; for (int i = 0; i < tags->numOfBound; ++i) { - SSchema* pTagSchema = &pSchema[tags->boundColumns[i] - 1]; // colId starts with 1 + SSchema* pTagSchema = &pSchema[tags->boundColumns[i] - 1]; // colId starts with 1 param.schema = pTagSchema; - SSmlKv *kv = taosArrayGetP(cols, i); - KvRowAppend(msg, kv->value, kv->valueLen, ¶m) ; + SSmlKv* kv = taosArrayGetP(cols, i); + KvRowAppend(msg, kv->value, kv->valueLen, ¶m); } - *row = tdGetKVRowFromBuilder(tagsBuilder); - if(*row == NULL){ + if (*row == NULL) { return TSDB_CODE_SML_INVALID_DATA; } tdSortKVRowByColIdx(*row); return TSDB_CODE_SUCCESS; } -int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SArray *colsSchema, SArray *cols, bool format, - STableMeta *pTableMeta, char *tableName, char *msgBuf, int16_t msgBufLen) { +int32_t smlBindData(void* handle, SArray* tags, SArray* colsFormat, SArray* colsSchema, SArray* cols, bool format, + STableMeta* pTableMeta, char* tableName, char* msgBuf, int16_t msgBufLen) { SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - SSmlExecHandle *smlHandle = (SSmlExecHandle *)handle; - SSchema* pTagsSchema = getTableTagSchema(pTableMeta); + SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle; + SSchema* pTagsSchema = getTableTagSchema(pTableMeta); setBoundColumnInfo(&smlHandle->tags, pTagsSchema, getNumOfTags(pTableMeta)); int ret = smlBoundColumnData(tags, &smlHandle->tags, pTagsSchema); - if(ret != TSDB_CODE_SUCCESS){ + if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "bound tags error"); return ret; } SKVRow row = NULL; ret = smlBuildTagRow(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &row, &pBuf); - if(ret != TSDB_CODE_SUCCESS){ + if (ret != TSDB_CODE_SUCCESS) { return ret; } buildCreateTbReq(&smlHandle->createTblReq, tableName, row, pTableMeta->suid); STableDataBlocks* pDataBlock = NULL; - ret = getDataBlockFromList(smlHandle->pBlockHash, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, pTableMeta, - &pDataBlock, NULL, &smlHandle->createTblReq); - if(ret != TSDB_CODE_SUCCESS){ + ret = getDataBlockFromList(smlHandle->pBlockHash, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), + getTableInfo(pTableMeta).rowSize, pTableMeta, &pDataBlock, NULL, &smlHandle->createTblReq); + if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "create data block error"); return ret; } @@ -1684,35 +1683,35 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SArray *cols SSchema* pSchema = getTableColumnSchema(pTableMeta); ret = smlBoundColumnData(colsSchema, &pDataBlock->boundColumnInfo, pSchema); - if(ret != TSDB_CODE_SUCCESS){ + if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "bound cols error"); return ret; } - int32_t extendedRowSize = getExtendedRowSize(pDataBlock); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; SRowBuilder* pBuilder = &pDataBlock->rowBuilder; - SMemParam param = {.rb = pBuilder}; + SMemParam param = {.rb = pBuilder}; initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo); int32_t rowNum = format ? taosArrayGetSize(colsFormat) : taosArrayGetSize(cols); - if(rowNum <= 0) { + if (rowNum <= 0) { return buildInvalidOperationMsg(&pBuf, "cols size <= 0"); } ret = allocateMemForSize(pDataBlock, extendedRowSize * rowNum); - if(ret != TSDB_CODE_SUCCESS){ + if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "allocate memory error"); return ret; } for (int32_t r = 0; r < rowNum; ++r) { STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header tdSRowResetBuf(pBuilder, row); - void *rowData = NULL; + void* rowData = NULL; size_t rowDataSize = 0; - if(format){ + if (format) { rowData = taosArrayGetP(colsFormat, r); rowDataSize = taosArrayGetSize(rowData); - }else{ + } else { rowData = taosArrayGetP(cols, r); } @@ -1723,19 +1722,20 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SArray *cols param.schema = pColSchema; getSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); - SSmlKv *kv = NULL; - if(format){ - if(j < rowDataSize){ + SSmlKv* kv = NULL; + if (format) { + if (j < rowDataSize) { kv = taosArrayGetP(rowData, j); - if (rowDataSize != spd->numOfBound && (kv->keyLen != strlen(pColSchema->name) || strncmp(kv->key, pColSchema->name, kv->keyLen) != 0)){ + if (rowDataSize != spd->numOfBound && + (kv->keyLen != strlen(pColSchema->name) || strncmp(kv->key, pColSchema->name, kv->keyLen) != 0)) { kv = NULL; - }else{ + } else { j++; } } - }else{ - void **p =taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); - if(p) kv = *p; + } else { + void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); + if (p) kv = *p; } if (!kv || kv->length == 0) { @@ -1744,7 +1744,7 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SArray *cols int32_t colLen = pColSchema->bytes; if (IS_VAR_DATA_TYPE(pColSchema->type)) { colLen = kv->length; - } else if(pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP){ + } else if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); } @@ -1753,7 +1753,7 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SArray *cols if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { TSKEY tsKey = TD_ROW_KEY(row); - checkTimestamp(pDataBlock, (const char *)&tsKey); + checkTimestamp(pDataBlock, (const char*)&tsKey); } } @@ -1770,7 +1770,7 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SArray *cols pDataBlock->size += extendedRowSize; } - SSubmitBlk *pBlocks = (SSubmitBlk *)(pDataBlock->pData); + SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, rowNum)) { return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than 32767"); } @@ -1778,25 +1778,24 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SArray *cols return TSDB_CODE_SUCCESS; } -void* smlInitHandle(SQuery *pQuery){ - SSmlExecHandle *handle = taosMemoryCalloc(1, sizeof(SSmlExecHandle)); - if(!handle) return NULL; +void* smlInitHandle(SQuery* pQuery) { + SSmlExecHandle* handle = taosMemoryCalloc(1, sizeof(SSmlExecHandle)); + if (!handle) return NULL; handle->pBlockHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); handle->pQuery = pQuery; return handle; } -void smlDestroyHandle(void *pHandle){ - if(!pHandle) return; - SSmlExecHandle *handle = (SSmlExecHandle *)pHandle; +void smlDestroyHandle(void* pHandle) { + if (!pHandle) return; + SSmlExecHandle* handle = (SSmlExecHandle*)pHandle; destroyBlockHashmap(handle->pBlockHash); taosMemoryFree(handle); } int32_t smlBuildOutput(void* handle, SHashObj* pVgHash) { - SSmlExecHandle *smlHandle = (SSmlExecHandle *)handle; + SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle; return qBuildStmtOutput(smlHandle->pQuery, pVgHash, smlHandle->pBlockHash); } // schemaless logic end - diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index cdcb2592a7..fddc8edb3a 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1886,6 +1886,9 @@ static int32_t checkDbKeepOption(STranslateContext* pCxt, SDatabaseOptions* pOpt TIME_UNIT_DAY != pVal->unit) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_UNIT, pVal->unit); } + if (!pVal->isDuration) { + pVal->datum.i = pVal->datum.i * 1440; + } } pOptions->keep[0] = getBigintFromValueNode((SValueNode*)nodesListGetNode(pOptions->pKeep, 0)); diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index cf364aba5c..8eaeffe6df 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -3,7 +3,7 @@ * * This program is free software: you can use, redistribute, and/or modify * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * or later ("AGPL"), AS published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or @@ -14,6 +14,7 @@ */ #include "parTestUtil.h" +#include "ttime.h" using namespace std; @@ -26,72 +27,226 @@ class ParserInitialCTest : public ParserDdlTest {}; TEST_F(ParserInitialCTest, createAccount) { useDb("root", "test"); - run("create account ac_wxy pass '123456'", TSDB_CODE_PAR_EXPRIE_STATEMENT); + run("CREATE ACCOUNT ac_wxy PASS '123456'", TSDB_CODE_PAR_EXPRIE_STATEMENT); } TEST_F(ParserInitialCTest, createBnode) { useDb("root", "test"); - run("create bnode on dnode 1"); + run("CREATE BNODE ON DNODE 1"); } +/* + * CREATE DATABASE [IF NOT EXISTS] db_name [database_options] + * + * database_options: + * database_option ... + * + * database_option: { + * BUFFER value + * | CACHELAST value + * | COMP {0 | 1 | 2} + * | DAYS value + * | FSYNC value + * | MAXROWS value + * | MINROWS value + * | KEEP value + * | PAGES value + * | PAGESIZE value + * | PRECISION {'ms' | 'us' | 'ns'} + * | REPLICA value + * | RETENTIONS ingestion_duration:keep_duration ... + * | STRICT value + * | WAL value + * | VGROUPS value + * | SINGLE_STABLE {0 | 1} + * } + */ TEST_F(ParserInitialCTest, createDatabase) { useDb("root", "test"); - run("create database wxy_db"); + SCreateDbReq expect = {0}; - run("create database if not exists wxy_db " - "cachelast 2 " - "comp 1 " - "days 100 " - "fsync 100 " - "maxrows 1000 " - "minrows 100 " - "keep 1440 " - "precision 'ms' " - "replica 3 " - "wal 2 " - "vgroups 100 " - "single_stable 0 " - "retentions 15s:7d,1m:21d,15m:5y"); + auto setCreateDbReqFunc = [&](const char* pDbname, int8_t igExists = 0) { + memset(&expect, 0, sizeof(SCreateDbReq)); + int32_t len = snprintf(expect.db, sizeof(expect.db), "0.%s", pDbname); + expect.db[len] = '\0'; + expect.ignoreExist = igExists; + expect.buffer = TSDB_DEFAULT_BUFFER_PER_VNODE; + expect.cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; + expect.compression = TSDB_DEFAULT_COMP_LEVEL; + expect.daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; + expect.fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; + expect.maxRows = TSDB_DEFAULT_MAXROWS_FBLOCK; + expect.minRows = TSDB_DEFAULT_MINROWS_FBLOCK; + expect.daysToKeep0 = TSDB_DEFAULT_KEEP; + expect.daysToKeep1 = TSDB_DEFAULT_KEEP; + expect.daysToKeep2 = TSDB_DEFAULT_KEEP; + expect.pages = TSDB_DEFAULT_PAGES_PER_VNODE; + expect.pageSize = TSDB_DEFAULT_PAGESIZE_PER_VNODE; + expect.precision = TSDB_DEFAULT_PRECISION; + expect.replications = TSDB_DEFAULT_DB_REPLICA; + expect.strict = TSDB_DEFAULT_DB_STRICT; + expect.walLevel = TSDB_DEFAULT_WAL_LEVEL; + expect.numOfVgroups = TSDB_DEFAULT_VN_PER_DB; + expect.numOfStables = TSDB_DEFAULT_DB_SINGLE_STABLE; + }; - run("create database if not exists wxy_db " - "days 100m " - "keep 1440m,300h,400d "); + auto setDbBufferFunc = [&](int32_t buffer) { expect.buffer = buffer; }; + auto setDbCachelastFunc = [&](int8_t CACHELAST) { expect.cacheLastRow = CACHELAST; }; + auto setDbCompressionFunc = [&](int8_t compressionLevel) { expect.compression = compressionLevel; }; + auto setDbDaysFunc = [&](int32_t daysPerFile) { expect.daysPerFile = daysPerFile; }; + auto setDbFsyncFunc = [&](int32_t fsyncPeriod) { expect.fsyncPeriod = fsyncPeriod; }; + auto setDbMaxRowsFunc = [&](int32_t maxRowsPerBlock) { expect.maxRows = maxRowsPerBlock; }; + auto setDbMinRowsFunc = [&](int32_t minRowsPerBlock) { expect.minRows = minRowsPerBlock; }; + auto setDbKeepFunc = [&](int32_t keep0, int32_t keep1 = 0, int32_t keep2 = 0) { + expect.daysToKeep0 = keep0; + expect.daysToKeep1 = 0 == keep1 ? expect.daysToKeep0 : keep1; + expect.daysToKeep2 = 0 == keep2 ? expect.daysToKeep1 : keep2; + }; + auto setDbPagesFunc = [&](int32_t pages) { expect.pages = pages; }; + auto setDbPageSizeFunc = [&](int32_t pagesize) { expect.pageSize = pagesize; }; + auto setDbPrecisionFunc = [&](int8_t precision) { expect.precision = precision; }; + auto setDbReplicaFunc = [&](int8_t replica) { expect.replications = replica; }; + auto setDbStrictaFunc = [&](int8_t strict) { expect.strict = strict; }; + auto setDbWalLevelFunc = [&](int8_t walLevel) { expect.walLevel = walLevel; }; + auto setDbVgroupsFunc = [&](int32_t numOfVgroups) { expect.numOfVgroups = numOfVgroups; }; + auto setDbSingleStableFunc = [&](int8_t singleStable) { expect.numOfStables = singleStable; }; + auto addDbRetentionFunc = [&](int64_t freq, int64_t keep, int8_t freqUnit, int8_t keepUnit) { + SRetention retention = {0}; + retention.freq = freq; + retention.keep = keep; + retention.freqUnit = freqUnit; + retention.keepUnit = keepUnit; + if (NULL == expect.pRetensions) { + expect.pRetensions = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SRetention)); + } + taosArrayPush(expect.pRetensions, &retention); + ++expect.numOfRetensions; + }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_DATABASE_STMT); + SCreateDbReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSCreateDbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); + + ASSERT_EQ(std::string(req.db), std::string(expect.db)); + ASSERT_EQ(req.numOfVgroups, expect.numOfVgroups); + ASSERT_EQ(req.numOfStables, expect.numOfStables); + ASSERT_EQ(req.buffer, expect.buffer); + ASSERT_EQ(req.pageSize, expect.pageSize); + ASSERT_EQ(req.pages, expect.pages); + ASSERT_EQ(req.daysPerFile, expect.daysPerFile); + ASSERT_EQ(req.daysToKeep0, expect.daysToKeep0); + ASSERT_EQ(req.daysToKeep1, expect.daysToKeep1); + ASSERT_EQ(req.daysToKeep2, expect.daysToKeep2); + ASSERT_EQ(req.minRows, expect.minRows); + ASSERT_EQ(req.maxRows, expect.maxRows); + ASSERT_EQ(req.fsyncPeriod, expect.fsyncPeriod); + ASSERT_EQ(req.walLevel, expect.walLevel); + ASSERT_EQ(req.precision, expect.precision); + ASSERT_EQ(req.compression, expect.compression); + ASSERT_EQ(req.replications, expect.replications); + ASSERT_EQ(req.strict, expect.strict); + ASSERT_EQ(req.cacheLastRow, expect.cacheLastRow); + ASSERT_EQ(req.ignoreExist, expect.ignoreExist); + ASSERT_EQ(req.numOfRetensions, expect.numOfRetensions); + if (expect.numOfRetensions > 0) { + ASSERT_EQ(taosArrayGetSize(req.pRetensions), expect.numOfRetensions); + ASSERT_EQ(taosArrayGetSize(req.pRetensions), taosArrayGetSize(expect.pRetensions)); + for (int32_t i = 0; i < expect.numOfRetensions; ++i) { + SRetention* pReten = (SRetention*)taosArrayGet(req.pRetensions, i); + SRetention* pExpectReten = (SRetention*)taosArrayGet(expect.pRetensions, i); + ASSERT_EQ(pReten->freq, pExpectReten->freq); + ASSERT_EQ(pReten->keep, pExpectReten->keep); + ASSERT_EQ(pReten->freqUnit, pExpectReten->freqUnit); + ASSERT_EQ(pReten->keepUnit, pExpectReten->keepUnit); + } + } + }); + + setCreateDbReqFunc("wxy_db"); + run("CREATE DATABASE wxy_db"); + + setCreateDbReqFunc("wxy_db", 1); + setDbBufferFunc(64); + setDbCachelastFunc(2); + setDbCompressionFunc(1); + setDbDaysFunc(100 * 1440); + setDbFsyncFunc(100); + setDbMaxRowsFunc(1000); + setDbMinRowsFunc(100); + setDbKeepFunc(1440 * 1440); + setDbPagesFunc(96); + setDbPageSizeFunc(8); + setDbPrecisionFunc(TSDB_TIME_PRECISION_NANO); + setDbReplicaFunc(3); + addDbRetentionFunc(15 * MILLISECOND_PER_SECOND, 7 * MILLISECOND_PER_DAY, TIME_UNIT_SECOND, TIME_UNIT_DAY); + addDbRetentionFunc(1 * MILLISECOND_PER_MINUTE, 21 * MILLISECOND_PER_DAY, TIME_UNIT_MINUTE, TIME_UNIT_DAY); + addDbRetentionFunc(15 * MILLISECOND_PER_MINUTE, 5, TIME_UNIT_MINUTE, TIME_UNIT_YEAR); + setDbStrictaFunc(1); + setDbWalLevelFunc(2); + setDbVgroupsFunc(100); + setDbSingleStableFunc(1); + run("CREATE DATABASE IF NOT EXISTS wxy_db " + "BUFFER 64 " + "CACHELAST 2 " + "COMP 1 " + "DAYS 100 " + "FSYNC 100 " + "MAXROWS 1000 " + "MINROWS 100 " + "KEEP 1440 " + "PAGES 96 " + "PAGESIZE 8 " + "PRECISION 'ns' " + "REPLICA 3 " + "RETENTIONS 15s:7d,1m:21d,15m:5y " + "STRICT 1 " + "WAL 2 " + "VGROUPS 100 " + "SINGLE_STABLE 1 "); + + setCreateDbReqFunc("wxy_db", 1); + setDbDaysFunc(100); + setDbKeepFunc(1440, 300 * 60, 400 * 1440); + run("CREATE DATABASE IF NOT EXISTS wxy_db " + "DAYS 100m " + "KEEP 1440m,300h,400d "); } TEST_F(ParserInitialCTest, createDnode) { useDb("root", "test"); - run("create dnode abc1 port 7000"); + run("CREATE DNODE abc1 PORT 7000"); - run("create dnode 1.1.1.1 port 9000"); + run("CREATE DNODE 1.1.1.1 PORT 9000"); } -// todo create function +// todo CREATE FUNCTION TEST_F(ParserInitialCTest, createIndexSma) { useDb("root", "test"); - run("create sma index index1 on t1 function(max(c1), min(c3 + 10), sum(c4)) INTERVAL(10s)"); + run("CREATE SMA INDEX index1 ON t1 FUNCTION(MAX(c1), MIN(c3 + 10), SUM(c4)) INTERVAL(10s)"); } TEST_F(ParserInitialCTest, createMnode) { useDb("root", "test"); - run("create mnode on dnode 1"); + run("CREATE MNODE ON DNODE 1"); } TEST_F(ParserInitialCTest, createQnode) { useDb("root", "test"); - run("create qnode on dnode 1"); + run("CREATE QNODE ON DNODE 1"); } TEST_F(ParserInitialCTest, createSnode) { useDb("root", "test"); - run("create snode on dnode 1"); + run("CREATE SNODE ON DNODE 1"); } TEST_F(ParserInitialCTest, createStable) { @@ -194,7 +349,7 @@ TEST_F(ParserInitialCTest, createStable) { addFieldToCreateStbReqFunc(true, "ts", TSDB_DATA_TYPE_TIMESTAMP); addFieldToCreateStbReqFunc(true, "c1", TSDB_DATA_TYPE_INT); addFieldToCreateStbReqFunc(false, "id", TSDB_DATA_TYPE_INT); - run("create stable t1(ts timestamp, c1 int) TAGS(id int)"); + run("CREATE STABLE t1(ts TIMESTAMP, c1 INT) TAGS(id INT)"); setCreateStbReqFunc("t1", 1, 0.1, 2, 100, "test create table"); addFieldToCreateStbReqFunc(true, "ts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0); @@ -227,80 +382,72 @@ TEST_F(ParserInitialCTest, createStable) { addFieldToCreateStbReqFunc(false, "a13", TSDB_DATA_TYPE_BOOL); addFieldToCreateStbReqFunc(false, "a14", TSDB_DATA_TYPE_NCHAR, 30 * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); addFieldToCreateStbReqFunc(false, "a15", TSDB_DATA_TYPE_VARCHAR, 50 + VARSTR_HEADER_SIZE); - run("create stable if not exists test.t1(" + run("CREATE STABLE IF NOT EXISTS test.t1(" "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " "c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " "c13 NCHAR(30), c14 VARCHAR(50)) " "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, " "a8 BINARY(20), a9 SMALLINT, a10 SMALLINT UNSIGNED COMMENT 'test column comment', a11 TINYINT, " "a12 TINYINT UNSIGNED, a13 BOOL, a14 NCHAR(30), a15 VARCHAR(50)) " - "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (min) FILE_FACTOR 0.1 DELAY 2"); + "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN) FILE_FACTOR 0.1 DELAY 2"); } TEST_F(ParserInitialCTest, createStream) { useDb("root", "test"); - run("create stream s1 as select * from t1"); + run("CREATE STREAM s1 AS SELECT * FROM t1"); - run("create stream if not exists s1 as select * from t1"); + run("CREATE STREAM IF NOT EXISTS s1 AS SELECT * FROM t1"); - run("create stream s1 into st1 as select * from t1"); + run("CREATE STREAM s1 INTO st1 AS SELECT * FROM t1"); - run("create stream if not exists s1 trigger window_close watermark 10s into st1 as select * from t1"); + run("CREATE STREAM IF NOT EXISTS s1 TRIGGER WINDOW_CLOSE WATERMARK 10s INTO st1 AS SELECT * FROM t1"); } TEST_F(ParserInitialCTest, createTable) { useDb("root", "test"); - run("create table t1(ts timestamp, c1 int)"); + run("CREATE TABLE t1(ts TIMESTAMP, c1 INT)"); - run("create table if not exists test.t1(" - "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), c8 " - "SMALLINT, " - "c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 " - "NCHAR(30), " - "c15 VARCHAR(50)) " + run("CREATE TABLE IF NOT EXISTS test.t1(" + "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " + "c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " + "c13 NCHAR(30), c15 VARCHAR(50)) " "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3)"); - run("create table if not exists test.t1(" - "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), c8 " - "SMALLINT, " - "c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 " - "NCHAR(30), " - "c15 VARCHAR(50)) " - "TAGS (tsa TIMESTAMP, a1 INT, a2 INT UNSIGNED, a3 BIGINT, a4 BIGINT UNSIGNED, " - "a5 FLOAT, a6 DOUBLE, a7 " - "BINARY(20), a8 SMALLINT, " - "a9 SMALLINT UNSIGNED COMMENT 'test column comment', a10 " - "TINYINT, a11 TINYINT UNSIGNED, a12 BOOL, a13 NCHAR(30), " - "a15 VARCHAR(50)) " - "TTL 100 COMMENT 'test create " - "table' SMA(c1, c2, c3) ROLLUP (min) FILE_FACTOR 0.1 DELAY 2"); + run("CREATE TABLE IF NOT EXISTS test.t1(" + "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " + "c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " + "c13 NCHAR(30), c14 VARCHAR(50)) " + "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, a8 BINARY(20), " + "a9 SMALLINT, a10 SMALLINT UNSIGNED COMMENT 'test column comment', a11 TINYINT, a12 TINYINT UNSIGNED, a13 BOOL, " + "a14 NCHAR(30), a15 VARCHAR(50)) " + "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN) FILE_FACTOR 0.1 DELAY 2"); - run("create table if not exists t1 using st1 tags(1, 'wxy')"); + run("CREATE TABLE IF NOT EXISTS t1 USING st1 TAGS(1, 'wxy')"); - run("create table " - "if not exists test.t1 using test.st1 (tag1, tag2) tags(1, 'abc') " - "if not exists test.t2 using test.st1 (tag1, tag2) tags(2, 'abc') " - "if not exists test.t3 using test.st1 (tag1, tag2) tags(3, 'abc') "); + run("CREATE TABLE " + "IF NOT EXISTS test.t1 USING test.st1 (tag1, tag2) TAGS(1, 'abc') " + "IF NOT EXISTS test.t2 USING test.st1 (tag1, tag2) TAGS(2, 'abc') " + "IF NOT EXISTS test.t3 USING test.st1 (tag1, tag2) TAGS(3, 'abc') "); } TEST_F(ParserInitialCTest, createTopic) { useDb("root", "test"); - run("create topic tp1 as select * from t1"); + run("CREATE TOPIC tp1 AS SELECT * FROM t1"); - run("create topic if not exists tp1 as select * from t1"); + run("CREATE TOPIC IF NOT EXISTS tp1 AS SELECT * FROM t1"); - run("create topic tp1 as test"); + run("CREATE TOPIC tp1 AS test"); - run("create topic if not exists tp1 as test"); + run("CREATE TOPIC IF NOT EXISTS tp1 AS test"); } TEST_F(ParserInitialCTest, createUser) { useDb("root", "test"); - run("create user wxy pass '123456'"); + run("CREATE USER wxy PASS '123456'"); } } // namespace ParserTest diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 6336377279..9ef4d80660 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -16,9 +16,10 @@ #include "planner.h" #include "planInt.h" +#include "scalar.h" typedef struct SCollectPlaceholderValuesCxt { - int32_t errCode; + int32_t errCode; SArray* pValues; } SCollectPlaceholderValuesCxt; @@ -144,9 +145,10 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { if (NULL == pVal->datum.p) { return TSDB_CODE_OUT_OF_MEMORY; } - + int32_t output = 0; - if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes, &output)) { + if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes, + &output)) { return errno; } varDataSetLen(pVal->datum.p, output); @@ -181,8 +183,8 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { } static EDealRes updatePlanQueryId(SNode* pNode, void* pContext) { - int64_t queryId = *(uint64_t *)pContext; - + int64_t queryId = *(uint64_t*)pContext; + if (QUERY_NODE_PHYSICAL_PLAN == nodeType(pNode)) { SQueryPlan* planNode = (SQueryPlan*)pNode; planNode->queryId = queryId; @@ -194,10 +196,130 @@ static EDealRes updatePlanQueryId(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId) { +static int32_t calcConstNode(SNode** pNode) { + if (NULL == *pNode) { + return TSDB_CODE_SUCCESS; + } + + SNode* pNew = NULL; + int32_t code = scalarCalculateConstants(*pNode, &pNew); + if (TSDB_CODE_SUCCESS == code) { + *pNode = pNew; + } + return code; +} + +static int32_t calcConstList(SNodeList* pList) { + SNode* pNode = NULL; + FOREACH(pNode, pList) { + SNode* pNew = NULL; + int32_t code = scalarCalculateConstants(pNode, &pNew); + if (TSDB_CODE_SUCCESS == code) { + REPLACE_NODE(pNew); + } else { + return code; + } + } + return TSDB_CODE_SUCCESS; +} + +static bool isEmptyResultCond(SNode** pCond) { + if (QUERY_NODE_VALUE != nodeType(*pCond)) { + return false; + } + if (((SValueNode*)*pCond)->datum.b) { + nodesDestroyNode(*pCond); + *pCond = NULL; + return false; + } + return true; +} + +static int32_t calcConstSpecificPhysiNode(SPhysiNode* pPhyNode) { + switch (nodeType(pPhyNode)) { + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: + case QUERY_NODE_PHYSICAL_PLAN_FILL: + return TSDB_CODE_SUCCESS; + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: + return calcConstList(((SProjectPhysiNode*)pPhyNode)->pProjections); + case QUERY_NODE_PHYSICAL_PLAN_JOIN: + return calcConstNode(&(((SJoinPhysiNode*)pPhyNode)->pOnConditions)); + case QUERY_NODE_PHYSICAL_PLAN_AGG: + return calcConstList(((SAggPhysiNode*)pPhyNode)->pExprs); + case QUERY_NODE_PHYSICAL_PLAN_SORT: + return calcConstList(((SSortPhysiNode*)pPhyNode)->pExprs); + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: + case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: + return calcConstList(((SWinodwPhysiNode*)pPhyNode)->pExprs); + case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + return calcConstList(((SPartitionPhysiNode*)pPhyNode)->pExprs); + default: + break; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t calcConstSubplan(SPhysiNode* pPhyNode, bool* pEmptyResult) { + int32_t code = calcConstNode(&pPhyNode->pConditions); + if (TSDB_CODE_SUCCESS == code) { + code = calcConstSpecificPhysiNode(pPhyNode); + } + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + *pEmptyResult = isEmptyResultCond(&pPhyNode->pConditions); + if (*pEmptyResult) { + return TSDB_CODE_SUCCESS; + } + + *pEmptyResult = true; + + bool subEmptyResult = false; + SNode* pChild = NULL; + FOREACH(pChild, pPhyNode->pChildren) { + code = calcConstSubplan((SPhysiNode*)pChild, &subEmptyResult); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + if (!subEmptyResult) { + *pEmptyResult = false; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t calcConstPhysiPlan(SQueryPlan* pPlan, bool* pEmptyResult) { + *pEmptyResult = true; + + bool subEmptyResult = false; + SNodeListNode* pNode = nodesListGetNode(pPlan->pSubplans, 0); + SNode* pSubplan = NULL; + FOREACH(pSubplan, pNode->pNodeList) { + int32_t code = calcConstSubplan(((SSubplan*)pSubplan)->pNode, pEmptyResult); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + if (!subEmptyResult) { + *pEmptyResult = false; + } + } + return TSDB_CODE_SUCCESS; +} + +int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId, + bool* pEmptyResult) { int32_t size = taosArrayGetSize(pPlan->pPlaceholderValues); int32_t code = 0; - + if (colIdx < 0) { for (int32_t i = 0; i < size; ++i) { code = setValueByBindParam((SValueNode*)taosArrayGetP(pPlan->pPlaceholderValues, i), pParams + i); @@ -214,9 +336,10 @@ int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colI if (colIdx < 0 || ((colIdx + 1) == size)) { nodesWalkPhysiPlan((SNode*)pPlan, updatePlanQueryId, &queryId); + code = calcConstPhysiPlan(pPlan, pEmptyResult); } - - return TSDB_CODE_SUCCESS; + + return code; } int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen) { From 510283290d5adcc31232726484c8beedcf2e4b1d Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 11 May 2022 19:49:04 +0800 Subject: [PATCH 38/94] fix: some problems of parser --- source/libs/function/src/builtins.c | 20 -------------------- tests/script/tsim/db/alter_option.sim | 4 ++-- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 5e3259c8fb..bf1dc091c6 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -977,26 +977,6 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = timezoneFunction, .finalizeFunc = NULL }, - // { - // .name = "_rowts", - // .type = FUNCTION_TYPE_ROWTS, - // .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC, - // .translateFunc = translateTimePseudoColumn, - // .getEnvFunc = getTimePseudoFuncEnv, - // .initFunc = NULL, - // .sprocessFunc = NULL, - // .finalizeFunc = NULL - // }, - // { - // .name = "_c0", - // .type = FUNCTION_TYPE_ROWTS, - // .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC, - // .translateFunc = translateTimePseudoColumn, - // .getEnvFunc = getTimePseudoFuncEnv, - // .initFunc = NULL, - // .sprocessFunc = NULL, - // .finalizeFunc = NULL - // }, { .name = "tbname", .type = FUNCTION_TYPE_TBNAME, diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 40882306c8..aeb04293f2 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -66,7 +66,7 @@ print ============= create database # | REPLICA value [1 | 3] # | WAL value [1 | 2] -sql create database db CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' REPLICA 3 WAL 2 VGROUPS 6 SINGLE_STABLE 1 +sql create database db CACHELAST 3 COMP 0 DAYS 240 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 PRECISION 'ns' REPLICA 3 WAL 2 VGROUPS 6 SINGLE_STABLE 1 sql show databases print rows: $rows print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 @@ -229,7 +229,7 @@ sql_error alter database db days 0 sql_error alter database db days 14400 # set over than keep print ============== modify keep -sql alter database db keep 3456000 +sql alter database db keep 2400 sql show databases print keep $data7_db if $data7_db != 3456000,3456000,3456000 then From ccd07312adb37aead4d6a8d7c3f1e4e2f7909327 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 11 May 2022 19:57:50 +0800 Subject: [PATCH 39/94] fix: some problems of parser --- tests/script/tsim/db/basic6.sim | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/script/tsim/db/basic6.sim b/tests/script/tsim/db/basic6.sim index 8075e54f9e..9075ebb2e8 100644 --- a/tests/script/tsim/db/basic6.sim +++ b/tests/script/tsim/db/basic6.sim @@ -15,8 +15,7 @@ $tb = $tbPrefix . $i print =============== step1 # quorum presicion -#sql create database $db vgroups 8 replica 1 days 2880 keep 3650 cache 32 blocks 12 minrows 80 maxrows 10000 wal 2 fsync 1000 comp 0 cachelast 2 precision 'us' -sql create database $db vgroups 8 replica 1 days 2880 keep 3650 minrows 80 maxrows 10000 wal 2 fsync 1000 comp 0 cachelast 2 precision 'us' +sql create database $db vgroups 8 replica 1 days 2 keep 10 minrows 80 maxrows 10000 wal 2 fsync 1000 comp 0 cachelast 2 precision 'us' sql show databases print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 @@ -38,7 +37,7 @@ endi if $data26 != 2880 then return -1 endi -if $data27 != 3650,3650,3650 then +if $data27 != 14400,14400,14400 then return -1 endi #if $data28 != 32 then @@ -67,7 +66,7 @@ print =============== step4 sql_error drop database $db print =============== step5 -sql create database $db replica 1 days 21600 keep 2160000 +sql create database $db replica 1 days 15 keep 1500 sql show databases print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 if $data20 != $db then From acbfe0c8ee82c1778fdcf5794fd50d431c93f043 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 11 May 2022 20:17:28 +0800 Subject: [PATCH 40/94] fix compile error --- source/dnode/vnode/src/vnd/vnodeSvr.c | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index ccaef3d433..fc2b6fe676 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -544,28 +544,7 @@ static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char while (true) { if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; if (pBlock == NULL) break; - tInitSubmitBlkIter(&msgIter, pBlock, &blkIter); - if (blkIter.row == NULL) continue; - if (!pSchema || (suid != msgIter.suid)) { - if (pSchema) { - taosMemoryFreeClear(pSchema); - } - pSchema = metaGetTbTSchema(pMeta, msgIter.suid, 0); // TODO: use the real schema - if (pSchema) { - suid = msgIter.suid; - } - } - if (!pSchema) { - printf("%s:%d no valid schema\n", tags, __LINE__); - continue; - } - char __tags[128] = {0}; - snprintf(__tags, 128, "%s: uid %" PRIi64 " ", tags, msgIter.uid); - while ((row = tGetSubmitBlkNext(&blkIter))) { - tdSRowPrint(row, pSchema, __tags); - } - } - + vnodeDebugPrintSingleSubmitMsg(pMeta, pBlock, &msgIter, tags); } From f57a652aa7591e9a93bc61cdbaf4fd91c9014450 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 May 2022 13:03:01 +0000 Subject: [PATCH 41/94] fix: create table coredump --- source/dnode/vnode/src/inc/meta.h | 5 +++++ source/dnode/vnode/src/meta/metaOpen.c | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 96feee3d7d..1e271bc8d8 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -37,6 +37,9 @@ typedef struct SMSmaCursor SMSmaCursor; // clang-format on // metaOpen ================== +int32_t metaRLock(SMeta* pMeta); +int32_t metaWLock(SMeta* pMeta); +int32_t metaULock(SMeta* pMeta); // metaEntry ================== int metaEncodeEntry(SEncoder* pCoder, const SMetaEntry* pME); @@ -57,6 +60,8 @@ int metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid); static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64(); } struct SMeta { + TdThreadRwlock lock; + char* path; SVnode* pVnode; TENV* pEnv; diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 85a49ffabd..8b3f555f4f 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -22,6 +22,9 @@ static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +static int32_t metaInitLock(SMeta *pMeta) { return taosThreadRwlockInit(&pMeta->lock, NULL); } +static int32_t metaDestroyLock(SMeta *pMeta) { return taosThreadRwlockDestroy(&pMeta->lock); } + int metaOpen(SVnode *pVnode, SMeta **ppMeta) { SMeta *pMeta = NULL; int ret; @@ -36,6 +39,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { return -1; } + metaInitLock(pMeta); pMeta->path = (char *)&pMeta[1]; sprintf(pMeta->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, VNODE_META_DIR); @@ -121,6 +125,7 @@ _err: if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb); if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb); if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); + metaDestroyLock(pMeta); taosMemoryFree(pMeta); return -1; } @@ -136,12 +141,19 @@ int metaClose(SMeta *pMeta) { if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb); if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb); if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); + metaDestroyLock(pMeta); taosMemoryFree(pMeta); } return 0; } +int32_t metaRLock(SMeta *pMeta) { return taosThreadRwlockRdlock(&pMeta->lock); } + +int32_t metaWLock(SMeta *pMeta) { return taosThreadRwlockWrlock(&pMeta->lock); } + +int32_t metaULock(SMeta *pMeta) { return taosThreadRwlockUnlock(&pMeta->lock); } + static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { STbDbKey *pTbDbKey1 = (STbDbKey *)pKey1; STbDbKey *pTbDbKey2 = (STbDbKey *)pKey2; From 953ded5662c7c17a87e62b73ff975bd7dfd5fdf6 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 11 May 2022 21:24:58 +0800 Subject: [PATCH 42/94] fix: commit after drop child table --- source/dnode/vnode/src/meta/metaQuery.c | 2 ++ source/dnode/vnode/src/tsdb/tsdbCommit.c | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 39c3a6c42d..68fbf01e28 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -264,6 +264,8 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { metaReaderClear(&mr); pSW = metaGetTableSchema(pMeta, quid, sver, 0); + if (!pSW) return NULL; + tdInitTSchemaBuilder(&sb, 0); for (int i = 0; i < pSW->nCols; i++) { pSchema = pSW->pSchema + i; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 5f54e0cb48..8aaa59c7f7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -311,7 +311,7 @@ static int tsdbNextCommitFid(SCommitH *pCommith) { for (int i = 0; i < pCommith->niters; i++) { SCommitIter *pIter = pCommith->iters + i; - // if (pIter->pTable == NULL || pIter->pIter == NULL) continue; + if (pIter->pTable == NULL || pIter->pIter == NULL) continue; TSKEY nextKey = tsdbNextIterKey(pIter->pIter); if (nextKey == TSDB_DATA_TIMESTAMP_NULL) { @@ -394,6 +394,9 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { ++fIter; } ++mIter; + } else if (pIter && !pIter->pTable) { + // When table already dropped during commit, pIter is not NULL but pIter->pTable is NULL. + ++mIter; // skip the table and do nothing } else if (pIdx) { if (tsdbMoveBlkIdx(pCommith, pIdx) < 0) { tsdbCloseCommitFile(pCommith, true); @@ -439,6 +442,7 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) { SCommitIter *pCommitIter; SSkipListNode *pNode; STbData *pTbData; + STSchema *pTSchema = NULL; pCommith->niters = SL_SIZE(pMem->pSlIdx); pCommith->iters = (SCommitIter *)taosMemoryCalloc(pCommith->niters, sizeof(SCommitIter)); @@ -462,10 +466,14 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) { pCommitIter->pIter = tSkipListCreateIter(pTbData->pData); tSkipListIterNext(pCommitIter->pIter); - pCommitIter->pTable = (STable *)taosMemoryMalloc(sizeof(STable)); - pCommitIter->pTable->uid = pTbData->uid; - pCommitIter->pTable->tid = pTbData->uid; - pCommitIter->pTable->pSchema = metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 0); + pTSchema = metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 0); // TODO: schema version + + if (pTSchema) { + pCommitIter->pTable = (STable *)taosMemoryMalloc(sizeof(STable)); + pCommitIter->pTable->uid = pTbData->uid; + pCommitIter->pTable->tid = pTbData->uid; + pCommitIter->pTable->pSchema = metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 0); + } } return 0; From af18343633b2825280141e28bf19ba958b32ef4d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 11 May 2022 21:32:44 +0800 Subject: [PATCH 43/94] enh: code optimization --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 8aaa59c7f7..be2895de53 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -382,6 +382,7 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { } else { break; } + if (pIter && pIter->pTable && (!pIdx || (pIter->pTable->uid <= pIdx->uid))) { if (tsdbCommitToTable(pCommith, mIter) < 0) { tsdbCloseCommitFile(pCommith, true); @@ -472,7 +473,7 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) { pCommitIter->pTable = (STable *)taosMemoryMalloc(sizeof(STable)); pCommitIter->pTable->uid = pTbData->uid; pCommitIter->pTable->tid = pTbData->uid; - pCommitIter->pTable->pSchema = metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 0); + pCommitIter->pTable->pSchema = pTSchema; //metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 0); } } From cc9983ab917ad5e11dbfe73294c86850d8f62dc1 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 11 May 2022 21:35:00 +0800 Subject: [PATCH 44/94] fix bug --- source/libs/scalar/src/filter.c | 3 +- source/libs/scalar/src/sclvector.c | 5 +- tests/script/api/batchprepare.c | 1123 +--------------------------- 3 files changed, 11 insertions(+), 1120 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index fe8aa77693..039e6693c2 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3512,7 +3512,8 @@ void fltConvertToTsValueNode(SFltTreeStat *stat, SValueNode* valueNode) { valueNode->datum.i = 0; } taosMemoryFree(timeStr); - + + valueNode->typeData = valueNode->datum.i; valueNode->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; valueNode->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 007f26ebc2..c9fcaeb32e 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -328,9 +328,10 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam* pOut, int32_t rowInd static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIndex) { int32_t len = 0; int32_t inputLen = varDataLen(buf); + int32_t outputMaxLen = (inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - char* t = taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); - /*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), pOut->columnData->info.bytes, &len); + char* t = taosMemoryCalloc(1, outputMaxLen); + /*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), outputMaxLen, &len); varDataSetLen(t, len); colDataAppend(pOut->columnData, rowIndex, t, false); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index bf38412b75..adf38015ae 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -92,7 +92,8 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos); int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos); int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos); int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos); -int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos); +int queryColumnTest(TAOS_STMT *stmt, TAOS *taos); +int queryMiscTest(TAOS_STMT *stmt, TAOS *taos); enum { TTYPE_INSERT = 1, @@ -152,7 +153,8 @@ CaseCfg gCase[] = { // 22 {"insert:AUTO1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, true, true, insertAUTOTest1, 10, 10, 2, 0, 0, 0, 1, -1}, - {"query:SUBT-FULL", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, querySUBTTest1, 10, 10, 1, 3, 0, 0, 1, 2}, + {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryColumnTest, 10, 10, 1, 3, 0, 0, 1, 2}, + {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, }; @@ -1749,7 +1751,7 @@ int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos) { /* select * from table */ -int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { +int queryColumnTest(TAOS_STMT *stmt, TAOS *taos) { BindData data = {0}; for (int32_t t = 0; t< gCurCase->tblNum; ++t) { @@ -1795,7 +1797,7 @@ int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { } /* value in query sql */ -int querySUBTTest2(TAOS_STMT *stmt, TAOS *taos) { +int queryMiscTest(TAOS_STMT *stmt, TAOS *taos) { BindData data = {0}; for (int32_t t = 0; t< gCurCase->tblNum; ++t) { @@ -1862,1119 +1864,6 @@ int errorSQLTest1(TAOS_STMT *stmt, TAOS *taos) { return 0; } - -#if 0 - -//1 tables 10 records -int stmt_funcb_autoctb1(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[10]; - int8_t v1[10]; - int16_t v2[10]; - int32_t v4[10]; - int64_t v8[10]; - float f4[10]; - double f8[10]; - char bin[10][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - - int *lb = taosMemoryMalloc(10 * sizeof(int)); - - TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - exit(1); - } - - int id = 0; - for (int zz = 0; zz < 1; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, tags); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - taosMemoryFree(tags); - - return 0; -} - - - - -//1 tables 10 records -int stmt_funcb_autoctb2(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[10]; - int8_t v1[10]; - int16_t v2[10]; - int32_t v4[10]; - int64_t v8[10]; - float f4[10]; - double f8[10]; - char bin[10][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - - int *lb = taosMemoryMalloc(10 * sizeof(int)); - - TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); - -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? using stb1 tags(1,true,2,3,4,5.0,6.0,'a','b') values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - exit(1); - } - - int id = 0; - for (int zz = 0; zz < 1; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, tags); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - taosMemoryFree(tags); - - return 0; -} - - - - - -//1 tables 10 records -int stmt_funcb_autoctb3(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[10]; - int8_t v1[10]; - int16_t v2[10]; - int32_t v4[10]; - int64_t v8[10]; - float f4[10]; - double f8[10]; - char bin[10][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - - int *lb = taosMemoryMalloc(10 * sizeof(int)); - - TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); - -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+0].buffer = v.b; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+1].buffer = v.v2; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+2].buffer = v.f4; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+3].buffer = v.bin; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = (uintptr_t *)lb; - } - - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? using stb1 tags(1,?,2,?,4,?,6.0,?,'b') values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - exit(1); - } - - int id = 0; - for (int zz = 0; zz < 1; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, tags); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - taosMemoryFree(tags); - - return 0; -} - - - - - - - -//1 tables 10 records -int stmt_funcb_autoctb4(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[10]; - int8_t v1[10]; - int16_t v2[10]; - int32_t v4[10]; - int64_t v8[10]; - float f4[10]; - double f8[10]; - char bin[10][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - - int *lb = taosMemoryMalloc(10 * sizeof(int)); - - TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*5); - -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 5; i+=5) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_INT; - params[i+2].buffer_length = sizeof(int32_t); - params[i+2].buffer = v.v4; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+3].buffer_length = sizeof(int64_t); - params[i+3].buffer = v.v8; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+4].buffer_length = sizeof(double); - params[i+4].buffer = v.f8; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+0].buffer = v.b; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+1].buffer = v.v2; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+2].buffer = v.f4; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+3].buffer = v.bin; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = (uintptr_t *)lb; - } - - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? using stb1 tags(1,?,2,?,4,?,6.0,?,'b') (ts,b,v4,v8,f8) values(?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - exit(1); - } - - int id = 0; - for (int zz = 0; zz < 1; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, tags); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 5); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - taosMemoryFree(tags); - - return 0; -} - - - - - -//1 tables 10 records -int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[10]; - int8_t v1[10]; - int16_t v2[10]; - int32_t v4[10]; - int64_t v8[10]; - float f4[10]; - double f8[10]; - char bin[10][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - - int *lb = taosMemoryMalloc(10 * sizeof(int)); - - TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); - -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+0].buffer = v.b; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+1].buffer = v.v2; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+2].buffer = v.f4; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+3].buffer = v.bin; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = (uintptr_t *)lb; - } - - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? using stb1 (id1,id2,id3,id4,id5,id6,id7,id8,id9) tags(1,?,2,?,4,?,6.0,?,'b') values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); - return -1; - } - - int id = 0; - for (int zz = 0; zz < 1; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, tags); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - taosMemoryFree(tags); - - return 0; -} - - - - - -//1 tables 10 records -int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[10]; - int8_t v1[10]; - int16_t v2[10]; - int32_t v4[10]; - int64_t v8[10]; - float f4[10]; - double f8[10]; - char bin[10][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - - int *lb = taosMemoryMalloc(10 * sizeof(int)); - - TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - exit(1); - } - - int id = 0; - for (int zz = 0; zz < 1; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, NULL); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. code:%s\n", taos_stmt_errstr(stmt)); - return -1; - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - taosMemoryFree(tags); - - return 0; -} - - - - - - - -//1 tables 10 records -int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[10]; - int8_t v1[10]; - int16_t v2[10]; - int32_t v4[10]; - int64_t v8[10]; - float f4[10]; - double f8[10]; - char bin[10][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - - int *lb = taosMemoryMalloc(10 * sizeof(int)); - - TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); - - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? using stb1 (id1,id2,id3,id4,id5,id6,id7,id8,id9) tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt)); - return -1; - //exit(1); - } - - int id = 0; - for (int zz = 0; zz < 1; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, NULL); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. code:0x%x\n", code); - return -1; - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - taosMemoryFree(tags); - - return 0; -} - - - - -//1 tables 10 records -int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[10]; - int8_t v1[10]; - int16_t v2[10]; - int32_t v4[10]; - int64_t v8[10]; - float f4[10]; - double f8[10]; - char bin[10][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - - int *lb = taosMemoryMalloc(10 * sizeof(int)); - - TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } - - int id = 0; - for (int zz = 0; zz < 1; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, tags); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. error:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } - - code = taos_stmt_bind_param_batch(stmt, params + id * 10); - if (code != 0) { - printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } - - code = taos_stmt_bind_param_batch(stmt, params + id * 10); - if (code != 0) { - printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); - return -1; - } - - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - taosMemoryFree(tags); - - return 0; -} - - - - - - -//1 tables 10 records -int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[10]; - int8_t v1[10]; - int16_t v2[10]; - int32_t v4[10]; - int64_t v8[10]; - float f4[10]; - double f8[10]; - char bin[10][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - - int *lb = taosMemoryMalloc(10 * sizeof(int)); - - TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(NULL, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:%s\n", taos_stmt_errstr(NULL)); - return -1; - } - - int id = 0; - for (int zz = 0; zz < 1; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname_tags(stmt, buf, tags); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname_tags. error:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } - - code = taos_stmt_bind_param_batch(stmt, params + id * 10); - if (code != 0) { - printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); - exit(1); - } - - code = taos_stmt_bind_param_batch(stmt, params + id * 10); - if (code != 0) { - printf("failed to execute taos_stmt_bind_param_batch. error:%s\n", taos_stmt_errstr(stmt)); - return -1; - } - - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - taosMemoryFree(tags); - - return 0; -} - -#endif - - void prepareCheckResultImpl(TAOS * taos, char *tname, bool printr, int expected, bool silent) { char sql[255] = "SELECT * FROM "; int32_t rows = 0; From 54458c1f59d72bdb1a90b47a6d90f6dad3879531 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 11 May 2022 21:40:10 +0800 Subject: [PATCH 45/94] enh: code optimization --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index be2895de53..031d200a66 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -311,7 +311,7 @@ static int tsdbNextCommitFid(SCommitH *pCommith) { for (int i = 0; i < pCommith->niters; i++) { SCommitIter *pIter = pCommith->iters + i; - if (pIter->pTable == NULL || pIter->pIter == NULL) continue; + // if (pIter->pTable == NULL || pIter->pIter == NULL) continue; TSKEY nextKey = tsdbNextIterKey(pIter->pIter); if (nextKey == TSDB_DATA_TIMESTAMP_NULL) { @@ -382,7 +382,7 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { } else { break; } - + if (pIter && pIter->pTable && (!pIdx || (pIter->pTable->uid <= pIdx->uid))) { if (tsdbCommitToTable(pCommith, mIter) < 0) { tsdbCloseCommitFile(pCommith, true); @@ -464,16 +464,16 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) { pTbData = (STbData *)pNode->pData; pCommitIter = pCommith->iters + i; - pCommitIter->pIter = tSkipListCreateIter(pTbData->pData); - tSkipListIterNext(pCommitIter->pIter); - - pTSchema = metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 0); // TODO: schema version + pTSchema = metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 0); // TODO: schema version if (pTSchema) { + pCommitIter->pIter = tSkipListCreateIter(pTbData->pData); + tSkipListIterNext(pCommitIter->pIter); + pCommitIter->pTable = (STable *)taosMemoryMalloc(sizeof(STable)); pCommitIter->pTable->uid = pTbData->uid; pCommitIter->pTable->tid = pTbData->uid; - pCommitIter->pTable->pSchema = pTSchema; //metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 0); + pCommitIter->pTable->pSchema = pTSchema; // metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 0); } } From a24318d4a62c890a86bbcfebd52399c786348ad1 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 11 May 2022 21:56:16 +0800 Subject: [PATCH 46/94] Merge branch '3.0' of github.com:taosdata/TDengine into test/chr/TD-14699 --- tests/system-test/1-insert/insertWithMoreVgroup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py index d8050c53c5..c9acc93c59 100644 --- a/tests/system-test/1-insert/insertWithMoreVgroup.py +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -346,8 +346,10 @@ class TDTestCase: return def test_case3(self): + self.taosBenchCreate("127.0.0.1","no","db1", "stb1", 1, 8, 1*10000) + self.taosBenchCreate("test209","no","db1", "stb1", 1, 8, 1*10000) + # self.taosBenchCreate("chenhaoran02","no","db1", "stb1", 1, 8, 1*10000) - self.taosBenchCreate("chenhaoran02","no","db1", "stb1", 1, 8, 1*1000) # self.taosBenchCreate("db1", "stb1", 4, 5, 100*10000) # self.taosBenchCreate("db1", "stb1", 1, 5, 100*10000) From 9882ae0a2170e3d14c8b5e1859b23d30569d020b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 May 2022 22:11:33 +0800 Subject: [PATCH 47/94] enh(index): update index filter UT --- source/libs/executor/test/executorTests.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 5ed960af82..c7ec0eece2 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -948,10 +948,6 @@ TEST(testCase, build_executor_tree_Test) { code = qCreateExecTask(&handle, 2, 1, plan, (void**)&pTaskInfo, &sinkHandle, OPTR_EXEC_MODEL_BATCH); ASSERT_EQ(code, 0); } -TEST(testCase, index_plan_test) { - // add later - EXPECT_EQ(0, 0); -} #if 0 TEST(testCase, inMem_sort_Test) { From b6c637dc6456af29a58b1fed2963ab25fda32a9d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 May 2022 22:13:46 +0800 Subject: [PATCH 48/94] enh(index): update index filter UT --- .../libs/executor/test/indexexcutorTests.cpp | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 source/libs/executor/test/indexexcutorTests.cpp diff --git a/source/libs/executor/test/indexexcutorTests.cpp b/source/libs/executor/test/indexexcutorTests.cpp new file mode 100644 index 0000000000..6293a24f5b --- /dev/null +++ b/source/libs/executor/test/indexexcutorTests.cpp @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#include "os.h" + +#include "executor.h" +#include "executorimpl.h" +#include "stub.h" +#include "taos.h" +#include "tcompare.h" +#include "tdatablock.h" +#include "tdef.h" +#include "trpc.h" +#include "tvariant.h" + +namespace { +SColumnInfo createColumnInfo(int32_t colId, int32_t type, int32_t bytes) { + SColumnInfo info = {0}; + info.colId = colId; + info.type = type; + info.bytes = bytes; + return info; +} + +int64_t sifLeftV = 21, sifRightV = 10; +double sifLeftVd = 21.0, sifRightVd = 10.0; + +void sifFreeDataBlock(void *block) { blockDataDestroy(*(SSDataBlock **)block); } + +void sifInitLogFile() { + const char * defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + qDebugFlag = 159; + strcpy(tsLogDir, "/tmp/sif"); + taosRemoveDir(tsLogDir); + taosMkDir(tsLogDir); + + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } +} + +void sifAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *slotId, bool newBlock, int32_t rows, + SColumnInfo *colInfo) { + if (newBlock) { + SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = 1; + res->info.rows = rows; + res->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + SColumnInfoData idata = {0}; + idata.info = *colInfo; + + taosArrayPush(res->pDataBlock, &idata); + taosArrayPush(pBlockList, &res); + + blockDataEnsureCapacity(res, rows); + + *dataBlockId = taosArrayGetSize(pBlockList) - 1; + res->info.blockId = *dataBlockId; + *slotId = 0; + } else { + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList); + res->info.numOfCols++; + SColumnInfoData idata = {0}; + idata.info = *colInfo; + + colInfoDataEnsureCapacity(&idata, 0, rows); + + taosArrayPush(res->pDataBlock, &idata); + + *dataBlockId = taosArrayGetSize(pBlockList) - 1; + *slotId = taosArrayGetSize(res->pDataBlock) - 1; + } +} + +void sifMakeValueNode(SNode **pNode, int32_t dataType, void *value) { + SNode * node = (SNode *)nodesMakeNode(QUERY_NODE_VALUE); + SValueNode *vnode = (SValueNode *)node; + vnode->node.resType.type = dataType; + + if (IS_VAR_DATA_TYPE(dataType)) { + vnode->datum.p = (char *)taosMemoryMalloc(varDataTLen(value)); + varDataCopy(vnode->datum.p, value); + vnode->node.resType.bytes = varDataTLen(value); + } else { + vnode->node.resType.bytes = tDataTypes[dataType].bytes; + assignVal((char *)nodesGetValueFromNode(vnode), (const char *)value, 0, dataType); + } + + *pNode = (SNode *)vnode; +} + +void sifMakeColumnNode(SNode **pNode, const char *db, const char *colName, EColumnType colType, uint8_t colValType) { + SNode * node = (SNode *)nodesMakeNode(QUERY_NODE_COLUMN); + SColumnNode *rnode = (SColumnNode *)node; + memcpy(rnode->dbName, db, strlen(db)); + memcpy(rnode->colName, colName, strlen(colName)); + + rnode->colType = colType; + rnode->node.resType.type = colValType; + + *pNode = (SNode *)rnode; +} + +void sifMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight) { + SNode * node = (SNode *)nodesMakeNode(QUERY_NODE_OPERATOR); + SOperatorNode *onode = (SOperatorNode *)node; + onode->node.resType.type = resType; + onode->node.resType.bytes = tDataTypes[resType].bytes; + + onode->opType = opType; + onode->pLeft = pLeft; + onode->pRight = pRight; + + *pNode = (SNode *)onode; +} + +void sifMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { + SNode * node = (SNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + SNodeListNode *lnode = (SNodeListNode *)node; + lnode->dataType.type = resType; + lnode->pNodeList = list; + + *pNode = (SNode *)lnode; +} + +void sifMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeList, int32_t nodeNum) { + SNode * node = (SNode *)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + SLogicConditionNode *onode = (SLogicConditionNode *)node; + onode->condType = opType; + onode->node.resType.type = TSDB_DATA_TYPE_BOOL; + onode->node.resType.bytes = sizeof(bool); + + onode->pParameterList = nodesMakeList(); + for (int32_t i = 0; i < nodeNum; ++i) { + nodesListAppend(onode->pParameterList, nodeList[i]); + } + + *pNode = (SNode *)onode; +} + +void sifMakeTargetNode(SNode **pNode, int16_t dataBlockId, int16_t slotId, SNode *snode) { + SNode * node = (SNode *)nodesMakeNode(QUERY_NODE_TARGET); + STargetNode *onode = (STargetNode *)node; + onode->pExpr = snode; + onode->dataBlockId = dataBlockId; + onode->slotId = slotId; + + *pNode = (SNode *)onode; +} + +} // namespace + +#if 1 +TEST(testCase, index_filter) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + + sifMakeColumnNode(&pLeft, "test", "col", COLUMN_TYPE_TAG, TSDB_DATA_TYPE_INT); + sifMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &sifRightV); + sifMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + nodesDestroyNode(res); +} + +TEST(testCase, index_filter_varify) { EXPECT_EQ(0, 0); } + +#endif + +#pragma GCC diagnostic pop From ae509e874e9a009f37b24477b1344fb91f83b9f8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 May 2022 14:20:14 +0000 Subject: [PATCH 49/94] fix: concurrent r/w meta --- source/dnode/vnode/src/meta/metaQuery.c | 4 ++++ source/dnode/vnode/src/meta/metaTable.c | 21 ++++++++++++++------- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 ++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 39c3a6c42d..1f5d6c60e4 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -19,9 +19,13 @@ void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { memset(pReader, 0, sizeof(*pReader)); pReader->flags = flags; pReader->pMeta = pMeta; + metaRLock(pMeta); } void metaReaderClear(SMetaReader *pReader) { + if (pReader->pMeta) { + metaULock(pReader->pMeta); + } tDecoderClear(&pReader->coder); tdbFree(pReader->pBuf); } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 7663047429..20248f39fb 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -439,29 +439,36 @@ _exit: } static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) { + metaWLock(pMeta); + // save to table.db - if (metaSaveToTbDb(pMeta, pME) < 0) return -1; + if (metaSaveToTbDb(pMeta, pME) < 0) goto _err; // update uid.idx - if (metaUpdateUidIdx(pMeta, pME) < 0) return -1; + if (metaUpdateUidIdx(pMeta, pME) < 0) goto _err; // update name.idx - if (metaUpdateNameIdx(pMeta, pME) < 0) return -1; + if (metaUpdateNameIdx(pMeta, pME) < 0) goto _err; if (pME->type == TSDB_CHILD_TABLE) { // update ctb.idx - if (metaUpdateCtbIdx(pMeta, pME) < 0) return -1; + if (metaUpdateCtbIdx(pMeta, pME) < 0) goto _err; // update tag.idx - if (metaUpdateTagIdx(pMeta, pME) < 0) return -1; + if (metaUpdateTagIdx(pMeta, pME) < 0) goto _err; } else { // update schema.db - if (metaSaveToSkmDb(pMeta, pME) < 0) return -1; + if (metaSaveToSkmDb(pMeta, pME) < 0) goto _err; } if (pME->type != TSDB_SUPER_TABLE) { - if (metaUpdateTtlIdx(pMeta, pME) < 0) return -1; + if (metaUpdateTtlIdx(pMeta, pME) < 0) goto _err; } + metaULock(pMeta); return 0; + +_err: + metaULock(pMeta); + return -1; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4638066935..2511e3a570 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3870,6 +3870,7 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch if (metaGetTableEntryByUid(&mr, uid) < 0) { tsdbError("%p failed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId); + metaReaderClear(&mr); terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; goto _error; } else { @@ -3880,6 +3881,7 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId); terrno = TSDB_CODE_OPS_NOT_SUPPORT; // basically, this error is caused by invalid sql issued by client + metaReaderClear(&mr); goto _error; } From 19e7f2207ba4a82d363393d04b0b2cfd6b03feee Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 May 2022 22:41:03 +0800 Subject: [PATCH 50/94] enh(index): update index filter UT --- .../libs/executor/test/indexexcutorTests.cpp | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/test/indexexcutorTests.cpp b/source/libs/executor/test/indexexcutorTests.cpp index 6293a24f5b..a105509c26 100644 --- a/source/libs/executor/test/indexexcutorTests.cpp +++ b/source/libs/executor/test/indexexcutorTests.cpp @@ -177,15 +177,45 @@ void sifMakeTargetNode(SNode **pNode, int16_t dataBlockId, int16_t slotId, SNode #if 1 TEST(testCase, index_filter) { - SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + sifMakeColumnNode(&pLeft, "test", "col", COLUMN_TYPE_TAG, TSDB_DATA_TYPE_INT); + sifMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &sifRightV); + sifMakeOpNode(&opNode, OP_LESS_THAN, TSDB_DATA_TYPE_INT, pLeft, pRight); + nodesDestroyNode(res); + } + { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + sifMakeColumnNode(&pLeft, "test", "col", COLUMN_TYPE_TAG, TSDB_DATA_TYPE_INT); + sifMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &sifRightV); + sifMakeOpNode(&opNode, OP_LESS_THAN, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); - sifMakeColumnNode(&pLeft, "test", "col", COLUMN_TYPE_TAG, TSDB_DATA_TYPE_INT); - sifMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &sifRightV); - sifMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); - nodesDestroyNode(res); + nodesDestroyNode(res); + } } -TEST(testCase, index_filter_varify) { EXPECT_EQ(0, 0); } +TEST(testCase, index_filter_varify) { + { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + sifMakeColumnNode(&pLeft, "test", "col", COLUMN_TYPE_TAG, TSDB_DATA_TYPE_INT); + sifMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &sifRightV); + sifMakeOpNode(&opNode, OP_LESS_THAN, TSDB_DATA_TYPE_INT, pLeft, pRight); + nodesDestroyNode(res); + + SIdxFltStatus st = idxGetFltStatus(opNode); + EXPECT_EQ(st, SFLT_ACCURATE_INDEX); + } + { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + sifMakeColumnNode(&pLeft, "test", "col", COLUMN_TYPE_TAG, TSDB_DATA_TYPE_INT); + sifMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &sifRightV); + sifMakeOpNode(&opNode, OP_LESS_THAN, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + SIdxFltStatus st = idxGetFltStatus(opNode); + EXPECT_EQ(st, SFLT_COARSE_INDEX); + nodesDestroyNode(res); + } +} #endif From f3bf12c7116c404c33686c590c50e95478cd1b03 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 11 May 2022 23:20:01 +0800 Subject: [PATCH 51/94] enh(index): update index filter UT --- source/libs/executor/inc/indexoperator.h | 16 +++++++- source/libs/executor/src/indexoperator.c | 6 +-- .../libs/executor/test/indexexcutorTests.cpp | 41 ++++++++++++++++--- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/source/libs/executor/inc/indexoperator.h b/source/libs/executor/inc/indexoperator.h index 9e67ac7f41..d033c63ef8 100644 --- a/source/libs/executor/inc/indexoperator.h +++ b/source/libs/executor/inc/indexoperator.h @@ -13,11 +13,23 @@ * along with this program. If not, see . */ -#include "filter.h" +#ifndef _INDEX_OPERATOR_H +#define _INDEX_OPERATOR_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "nodes.h" #include "tglobal.h" typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus; SIdxFltStatus idxGetFltStatus(SNode *pFilterNode); // construct tag filter operator later -int32_t doFilterTag(const SNode *pFilterNode, SArray *resutl); +int32_t doFilterTag(const SNode *pFilterNode, SArray *result); + +#ifdef __cplusplus +} +#endif + +#endif /*INDEX_OPERATOR_*/ diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index fe30ebb2ea..c17fcacf1f 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -583,7 +583,7 @@ int32_t doFilterTag(const SNode *pFilterNode, SArray *result) { SFilterInfo *filter = NULL; // todo move to the initialization function - SIF_ERR_RET(filterInitFromNode((SNode *)pFilterNode, &filter, 0)); + // SIF_ERR_RET(filterInitFromNode((SNode *)pFilterNode, &filter, 0)); SIFParam param = {0}; SIF_ERR_RET(sifCalculate((SNode *)pFilterNode, ¶m)); @@ -598,9 +598,9 @@ SIdxFltStatus idxGetFltStatus(SNode *pFilterNode) { if (pFilterNode == NULL) { return SFLT_NOT_INDEX; } - SFilterInfo *filter = NULL; + // SFilterInfo *filter = NULL; // todo move to the initialization function - SIF_ERR_RET(filterInitFromNode((SNode *)pFilterNode, &filter, 0)); + // SIF_ERR_RET(filterInitFromNode((SNode *)pFilterNode, &filter, 0)); SIF_ERR_RET(sifGetFltHint((SNode *)pFilterNode, &st)); return st; diff --git a/source/libs/executor/test/indexexcutorTests.cpp b/source/libs/executor/test/indexexcutorTests.cpp index a105509c26..5f1bff45a3 100644 --- a/source/libs/executor/test/indexexcutorTests.cpp +++ b/source/libs/executor/test/indexexcutorTests.cpp @@ -23,10 +23,12 @@ #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" -#include "os.h" #include "executor.h" #include "executorimpl.h" +#include "indexoperator.h" +#include "os.h" + #include "stub.h" #include "taos.h" #include "tcompare.h" @@ -181,15 +183,24 @@ TEST(testCase, index_filter) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; sifMakeColumnNode(&pLeft, "test", "col", COLUMN_TYPE_TAG, TSDB_DATA_TYPE_INT); sifMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &sifRightV); - sifMakeOpNode(&opNode, OP_LESS_THAN, TSDB_DATA_TYPE_INT, pLeft, pRight); + sifMakeOpNode(&opNode, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_INT, pLeft, pRight); + SArray *result = taosArrayInit(4, sizeof(uint64_t)); + doFilterTag(opNode, result); + EXPECT_EQ(1, taosArrayGetSize(result)); + taosArrayDestroy(result); nodesDestroyNode(res); } { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; sifMakeColumnNode(&pLeft, "test", "col", COLUMN_TYPE_TAG, TSDB_DATA_TYPE_INT); sifMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &sifRightV); - sifMakeOpNode(&opNode, OP_LESS_THAN, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + sifMakeOpNode(&opNode, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + SArray *result = taosArrayInit(4, sizeof(uint64_t)); + doFilterTag(opNode, result); + EXPECT_EQ(1, taosArrayGetSize(result)); + + taosArrayDestroy(result); nodesDestroyNode(res); } } @@ -199,7 +210,7 @@ TEST(testCase, index_filter_varify) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; sifMakeColumnNode(&pLeft, "test", "col", COLUMN_TYPE_TAG, TSDB_DATA_TYPE_INT); sifMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &sifRightV); - sifMakeOpNode(&opNode, OP_LESS_THAN, TSDB_DATA_TYPE_INT, pLeft, pRight); + sifMakeOpNode(&opNode, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_INT, pLeft, pRight); nodesDestroyNode(res); SIdxFltStatus st = idxGetFltStatus(opNode); @@ -209,7 +220,27 @@ TEST(testCase, index_filter_varify) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; sifMakeColumnNode(&pLeft, "test", "col", COLUMN_TYPE_TAG, TSDB_DATA_TYPE_INT); sifMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &sifRightV); - sifMakeOpNode(&opNode, OP_LESS_THAN, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + sifMakeOpNode(&opNode, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + SIdxFltStatus st = idxGetFltStatus(opNode); + EXPECT_EQ(st, SFLT_COARSE_INDEX); + nodesDestroyNode(res); + } + { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + sifMakeColumnNode(&pLeft, "test", "col", COLUMN_TYPE_TAG, TSDB_DATA_TYPE_INT); + sifMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &sifRightV); + sifMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_INT, pLeft, pRight); + nodesDestroyNode(res); + + SIdxFltStatus st = idxGetFltStatus(opNode); + EXPECT_EQ(st, SFLT_ACCURATE_INDEX); + } + { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + sifMakeColumnNode(&pLeft, "test", "col", COLUMN_TYPE_TAG, TSDB_DATA_TYPE_INT); + sifMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &sifRightV); + sifMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); SIdxFltStatus st = idxGetFltStatus(opNode); EXPECT_EQ(st, SFLT_COARSE_INDEX); From 453d105a08f8c390e780dafd88238d31e194a659 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 12 May 2022 07:54:45 +0800 Subject: [PATCH 52/94] enh(sync): add error log, linux api error, %X --- source/libs/sync/src/syncRaftLog.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 3ee952fcdf..8aeb9c4856 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -62,7 +62,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { const char *errStr = tstrerror(err); int32_t linuxErr = errno; const char *linuxErrMsg = strerror(errno); - sError("walWriteWithSyncInfo error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, linuxErr, linuxErrMsg); + sError("walWriteWithSyncInfo error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); ASSERT(0); } //assert(code == 0); @@ -83,7 +83,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { const char *errStr = tstrerror(err); int32_t linuxErr = errno; const char *linuxErrMsg = strerror(errno); - sError("walReadWithHandle error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, linuxErr, linuxErrMsg); + sError("walReadWithHandle error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); ASSERT(0); } //assert(walReadWithHandle(pWalHandle, index) == 0); @@ -119,7 +119,7 @@ int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { const char *errStr = tstrerror(err); int32_t linuxErr = errno; const char *linuxErrMsg = strerror(errno); - sError("walRollback error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, linuxErr, linuxErrMsg); + sError("walRollback error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); ASSERT(0); } return 0; // to avoid compiler error @@ -152,7 +152,7 @@ int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { const char *errStr = tstrerror(err); int32_t linuxErr = errno; const char *linuxErrMsg = strerror(errno); - sError("walCommit error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, linuxErr, linuxErrMsg); + sError("walCommit error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); ASSERT(0); } return 0; // to avoid compiler error From 4765cb9e9a1254366d4bb9afe06703d45475039b Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 12 May 2022 09:42:53 +0800 Subject: [PATCH 53/94] test:modify testcase that using taosBenchmark to test multi-process table building and data inserting --- tests/system-test/1-insert/insertWithMoreVgroup.py | 2 +- tests/system-test/1-insert/manyVgroups.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py index c9acc93c59..d3da4f2c59 100644 --- a/tests/system-test/1-insert/insertWithMoreVgroup.py +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -347,7 +347,7 @@ class TDTestCase: def test_case3(self): self.taosBenchCreate("127.0.0.1","no","db1", "stb1", 1, 8, 1*10000) - self.taosBenchCreate("test209","no","db1", "stb1", 1, 8, 1*10000) + # self.taosBenchCreate("test209","no","db2", "stb2", 1, 8, 1*10000) # self.taosBenchCreate("chenhaoran02","no","db1", "stb1", 1, 8, 1*10000) diff --git a/tests/system-test/1-insert/manyVgroups.json b/tests/system-test/1-insert/manyVgroups.json index 5487dff708..e6719aedc9 100644 --- a/tests/system-test/1-insert/manyVgroups.json +++ b/tests/system-test/1-insert/manyVgroups.json @@ -29,8 +29,8 @@ "batch_create_tbl_num": 50000, "data_source": "rand", "insert_mode": "taosc", - "insert_rows": 0, - "interlace_rows": 0, + "insert_rows": 10, + "interlace_rows": 100000, "insert_interval": 0, "max_sql_len": 10000000, "disorder_ratio": 0, From d37d4e5b5eda68ae568c919c57a38dba7c4a2180 Mon Sep 17 00:00:00 2001 From: "slzhou@taodata.com" Date: Thu, 12 May 2022 09:57:43 +0800 Subject: [PATCH 54/94] error processing of executor when calling aggregate function --- source/libs/executor/inc/executorimpl.h | 4 +- source/libs/executor/src/executorimpl.c | 39 +++++++++++++------ source/libs/executor/src/groupoperator.c | 8 ++-- source/libs/executor/src/timewindowoperator.c | 26 ++++++------- 4 files changed, 47 insertions(+), 30 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 93e81aa70e..4881f23134 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -616,10 +616,10 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, size_t keyBufSize, const char* pkey); void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows); -void doBuildResultDatablock(SOptrBasicInfo *pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf); +void doBuildResultDatablock(SExecTaskInfo *taskInfo, SOptrBasicInfo *pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf); void finalizeMultiTupleQueryResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); -void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, +void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order); int32_t setGroupResultOutputBuf(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type, int16_t bytes, int32_t groupId, SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, SAggSupporter* pAggSup); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 33f0c440ec..9c285856d3 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -155,7 +155,7 @@ SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, void operatorDummyCloseFn(void* param, int32_t numOfCols) {} -static int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, +static int32_t doCopyToSDataBlock(SExecTaskInfo *taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType, int32_t* rowCellOffset, SqlFunctionCtx* pCtx); @@ -579,7 +579,7 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); } -void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, +void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) { for (int32_t k = 0; k < numOfOutput; ++k) { pCtx[k].startTs = pWin->skey; @@ -618,9 +618,14 @@ void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pEntryInfo->numOfRes = 1; continue; } - + int32_t code = TSDB_CODE_SUCCESS; if (functionNeedToExecute(&pCtx[k]) && pCtx[k].fpSet.process != NULL) { - pCtx[k].fpSet.process(&pCtx[k]); + code = pCtx[k].fpSet.process(&pCtx[k]); + if (code != TSDB_CODE_SUCCESS) { + qError("%s apply functions error, code: %s", GET_TASKID(taskInfo), tstrerror(code)); + taskInfo->code = code; + longjmp(taskInfo->env, code); + } } // restore it @@ -806,7 +811,13 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunction // this can be set during create the struct // todo add a dummy funtion to avoid process check if (pCtx[k].fpSet.process != NULL) { - pCtx[k].fpSet.process(&pCtx[k]); + int32_t code = pCtx[k].fpSet.process(&pCtx[k]); + if (code != TSDB_CODE_SUCCESS) { + qError("%s call aggregate function error happens, code : %s", + GET_TASKID(pOperator->pTaskInfo), tstrerror(code)); + pOperator->pTaskInfo->code = code; + longjmp(pOperator->pTaskInfo->env, code); + } } } } @@ -2176,7 +2187,7 @@ void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* p * @param pQInfo * @param result */ -int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, +int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType, int32_t* rowCellOffset, SqlFunctionCtx* pCtx) { int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); int32_t numOfResult = pBlock->info.rows; // there are already exists result rows @@ -2215,8 +2226,14 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased int32_t slotId = pExprInfo[j].base.resSchema.slotId; pCtx[j].resultInfo = getResultCell(pRow, j, rowCellOffset); - if (pCtx[j].fpSet.process) { - pCtx[j].fpSet.finalize(&pCtx[j], pBlock); + if (pCtx[j].fpSet.finalize) { + int32_t code = TSDB_CODE_SUCCESS; + code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); + if (code != TSDB_CODE_SUCCESS) { + qError("%s build result data block error, code %s", GET_TASKID(taskInfo), tstrerror(code)); + taskInfo->code = code; + longjmp(taskInfo->env, code); + } } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { // do nothing, todo refactor } else { @@ -2243,7 +2260,7 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased return 0; } -void doBuildResultDatablock(SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, +void doBuildResultDatablock(SExecTaskInfo *taskInfo, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf) { assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); @@ -2257,7 +2274,7 @@ void doBuildResultDatablock(SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo } int32_t orderType = TSDB_ORDER_ASC; - doCopyToSDataBlock(pBlock, pExprInfo, pBuf, pGroupResInfo, orderType, rowCellOffset, pCtx); + doCopyToSDataBlock(taskInfo, pBlock, pExprInfo, pBuf, pGroupResInfo, orderType, rowCellOffset, pCtx); // add condition (pBlock->info.rows >= 1) just to runtime happy blockDataUpdateTsWindow(pBlock); @@ -3749,7 +3766,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { } blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pInfo, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf); + doBuildResultDatablock(pTaskInfo, pInfo, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf); if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pAggInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index e3a507bf7c..5d22c13ec6 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -234,7 +234,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } int32_t rowIndex = j - num; - doApplyFunctions(pCtx, &w, NULL, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfExprs, TSDB_ORDER_ASC); + doApplyFunctions(pTaskInfo, pCtx, &w, NULL, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfExprs, TSDB_ORDER_ASC); // assign the group keys or user input constant values if required doAssignGroupKeys(pCtx, pOperator->numOfExprs, pBlock->info.rows, rowIndex); @@ -252,7 +252,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } int32_t rowIndex = pBlock->info.rows - num; - doApplyFunctions(pCtx, &w, NULL, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfExprs, TSDB_ORDER_ASC); + doApplyFunctions(pTaskInfo, pCtx, &w, NULL, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfExprs, TSDB_ORDER_ASC); doAssignGroupKeys(pCtx, pOperator->numOfExprs, pBlock->info.rows, rowIndex); } } @@ -268,7 +268,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pTaskInfo, &pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); if (pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -317,7 +317,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, false); while(1) { - doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pTaskInfo, &pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); doFilter(pInfo->pCondition, pRes); bool hasRemain = hasRemainDataInCurrentGroup(&pInfo->groupResInfo); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 738f4821bd..0f3b1bda20 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -703,7 +703,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe pInfo->order, false); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); - doApplyFunctions(pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); STimeWindow nextWin = win; @@ -740,7 +740,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe pInfo->order, false); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } @@ -855,7 +855,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); - doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); // here we start a new session window @@ -874,7 +874,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); - doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } @@ -888,7 +888,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pTaskInfo, pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); return NULL; @@ -921,7 +921,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pTaskInfo, pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -948,7 +948,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { } blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); - doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pTaskInfo, &pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); if (pBlock->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -998,7 +998,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pOperator->pTaskInfo, &pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -1035,7 +1035,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pOperator->pTaskInfo, &pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); // TODO: remove for stream /*ASSERT(pInfo->binfo.pRes->info.rows > 0);*/ @@ -1233,7 +1233,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator // pInfo->numOfRows data belong to the current session window updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); - doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); // here we start a new session window @@ -1252,7 +1252,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); - doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } @@ -1265,7 +1265,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pOperator->pTaskInfo, pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); return NULL; @@ -1298,7 +1298,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); + doBuildResultDatablock(pOperator->pTaskInfo, pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } From 85ee4f7df60f90d7154817d253368e22ee88b533 Mon Sep 17 00:00:00 2001 From: "slzhou@taodata.com" Date: Thu, 12 May 2022 10:43:54 +0800 Subject: [PATCH 55/94] fix finalize function call return value >=0 not error, <0 error --- source/libs/executor/src/executorimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 9c285856d3..a5bc1fdf58 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2229,7 +2229,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprIn if (pCtx[j].fpSet.finalize) { int32_t code = TSDB_CODE_SUCCESS; code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); - if (code != TSDB_CODE_SUCCESS) { + if (TAOS_FAILED(code)) { qError("%s build result data block error, code %s", GET_TASKID(taskInfo), tstrerror(code)); taskInfo->code = code; longjmp(taskInfo->env, code); From 0a1e3f144c54f6eb3ff5c13a115a478281bbd06f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 12 May 2022 11:09:05 +0800 Subject: [PATCH 56/94] fix taosd crash issue --- source/client/inc/clientStmt.h | 1 + source/client/src/clientStmt.c | 10 +++++++--- source/libs/parser/src/parInsert.c | 9 ++++++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index e5efafc214..ae27e611cb 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -71,6 +71,7 @@ typedef struct SStmtBindInfo { typedef struct SStmtExecInfo { int32_t affectedRows; + bool emptyRes; SRequestObj* pRequest; SHashObj* pVgHash; SHashObj* pBlockHash; diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index a7e50b2044..29f965fd74 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -279,6 +279,7 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) { } pStmt->exec.autoCreateTbl = false; + pStmt->exec.emptyRes = false; if (keepTable) { return TSDB_CODE_SUCCESS; @@ -628,8 +629,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { STMT_ERR_RET(stmtRestoreQueryFields(pStmt)); } - bool emptyResult = false; - STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx, pStmt->exec.pRequest->requestId, &emptyResult)); + STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx, pStmt->exec.pRequest->requestId, &pStmt->exec.emptyRes)); } STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); @@ -736,7 +736,11 @@ int stmtExec(TAOS_STMT *stmt) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); if (STMT_TYPE_QUERY == pStmt->sql.type) { - scheduleQuery(pStmt->exec.pRequest, pStmt->sql.pQueryPlan, pStmt->sql.nodeList, NULL); + if (pStmt->exec.emptyRes) { + pStmt->exec.pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + } else { + scheduleQuery(pStmt->exec.pRequest, pStmt->sql.pQueryPlan, pStmt->sql.nodeList, NULL); + } } else { STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash)); launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true, (autoCreateTbl ? (void**)&pRsp : NULL)); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 3d069257c9..d4f547f2cf 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1065,6 +1065,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { int32_t tbNum = 0; char tbFName[TSDB_TABLE_FNAME_LEN]; bool autoCreateTbl = false; + STableMeta *pMeta = NULL; // for each table while (1) { @@ -1119,10 +1120,12 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, tbFName, strlen(tbFName), TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL, &pCxt->createTblReq)); - + pMeta = pCxt->pTableMeta; + pCxt->pTableMeta = NULL; + if (TK_NK_LP == sToken.type) { // pSql -> field1_name, ...) - CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta))); + CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pMeta))); NEXT_TOKEN(pCxt->pSql, sToken); } @@ -1158,7 +1161,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } memcpy(tags, &pCxt->tags, sizeof(pCxt->tags)); - (*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pTableMeta, tags, tbFName, autoCreateTbl, pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj); + (*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pMeta, tags, tbFName, autoCreateTbl, pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj); memset(&pCxt->tags, 0, sizeof(pCxt->tags)); pCxt->pVgroupsHashObj = NULL; From 7ab2e76c3535b12f7f7e75b83f3fe8e4c2ed45dd Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 12 May 2022 11:31:56 +0800 Subject: [PATCH 57/94] feat(tmq): support get tb name --- example/src/tmq.c | 9 +++++++-- include/client/taos.h | 5 +---- include/common/tmsg.h | 10 ++++++++++ source/client/src/tmq.c | 13 +++++++++++++ source/dnode/mnode/impl/src/mndTopic.c | 4 ++-- source/dnode/vnode/src/inc/meta.h | 10 +++++----- source/dnode/vnode/src/tq/tq.c | 26 ++++++++++++++++++++++++++ 7 files changed, 64 insertions(+), 13 deletions(-) diff --git a/example/src/tmq.c b/example/src/tmq.c index 976d658fa6..2ee91c254c 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -32,6 +32,11 @@ static void msg_process(TAOS_RES* msg) { int32_t numOfFields = taos_field_count(msg); taos_print_row(buf, row, fields, numOfFields); printf("%s\n", buf); + + const char* tbName = tmq_get_table_name(msg); + if (tbName) { + printf("from tb: %s\n", tbName); + } } } @@ -101,8 +106,8 @@ int32_t create_topic() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create topic topic_ctb_column as abc1"); - /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1");*/ + /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/ + pRes = taos_query(pConn, "create topic topic_ctb_column with table as select ts, c1, c2, c3 from st1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/include/client/taos.h b/include/client/taos.h index 26d4d18234..486d5f5fef 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -257,10 +257,7 @@ DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_co DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); -// TODO -#if 0 -DLL_EXPORT char *tmq_get_table_name(TAOS_RES *res); -#endif +DLL_EXPORT const char *tmq_get_table_name(TAOS_RES *res); #if 0 DLL_EXPORT int64_t tmq_get_request_offset(tmq_message_t *message); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b50367af03..d34892a278 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2480,6 +2480,10 @@ static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRsp->blockSchema, i); tlen += taosEncodeSSchemaWrapper(buf, pSW); } + if (pRsp->withTbName) { + char* tbName = (char*)taosArrayGetP(pRsp->blockTbName, i); + tlen += taosEncodeString(buf, tbName); + } } } return tlen; @@ -2492,6 +2496,7 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p buf = taosDecodeFixedI32(buf, &pRsp->blockNum); pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*)); pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*)); + pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*)); pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*)); if (pRsp->blockNum != 0) { buf = taosDecodeFixedI8(buf, &pRsp->withTbName); @@ -2510,6 +2515,11 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p buf = taosDecodeSSchemaWrapper(buf, pSW); taosArrayPush(pRsp->blockSchema, &pSW); } + if (pRsp->withTbName) { + char* name = NULL; + buf = taosDecodeString(buf, &name); + taosArrayPush(pRsp->blockTbName, &name); + } } } return (void*)buf; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 0ce689f19c..b42f072e54 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -1346,3 +1346,16 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) { return -1; } } + +const char* tmq_get_table_name(TAOS_RES* res) { + if (TD_RES_TMQ(res)) { + SMqRspObj* pRspObj = (SMqRspObj*)res; + if (!pRspObj->rsp.withTbName || pRspObj->rsp.blockTbName == NULL || pRspObj->resIter < 0 || + pRspObj->resIter >= pRspObj->rsp.blockNum) { + return NULL; + } + const char* name = taosArrayGetP(pRspObj->rsp.blockTbName, pRspObj->resIter); + return name; + } + return NULL; +} diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 41d4d5f406..00379ecda1 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -297,8 +297,8 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq topicObj.ast = strdup(pCreate->ast); topicObj.astLen = strlen(pCreate->ast) + 1; topicObj.subType = TOPIC_SUB_TYPE__TABLE; - topicObj.withTbName = 0; - topicObj.withSchema = 0; + topicObj.withTbName = pCreate->withTbName; + topicObj.withSchema = pCreate->withSchema; SNode *pAst = NULL; if (nodesStringToNode(pCreate->ast, &pAst) != 0) { diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 96feee3d7d..6dd9ac0fed 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -111,10 +111,10 @@ int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur); // SMetaDB int metaOpenDB(SMeta* pMeta); void metaCloseDB(SMeta* pMeta); -int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); -int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); -int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg); -int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid); +// int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); +int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); +int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg); +int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid); #endif #endif @@ -123,4 +123,4 @@ int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid); } #endif -#endif /*_TD_VNODE_META_H_*/ \ No newline at end of file +#endif /*_TD_VNODE_META_H_*/ diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4b9551b250..29fabc0f9f 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -427,9 +427,12 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqDataBlkRsp rsp = {0}; rsp.reqOffset = pReq->currentOffset; rsp.withSchema = pExec->withSchema; + rsp.withTbName = pExec->withTbName; + rsp.blockData = taosArrayInit(0, sizeof(void*)); rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t)); rsp.blockSchema = taosArrayInit(0, sizeof(void*)); + rsp.blockTbName = taosArrayInit(0, sizeof(void*)); while (1) { consumerEpoch = atomic_load_32(&pExec->epoch); @@ -535,6 +538,18 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { taosArrayPush(rsp.blockSchema, &pSW); } + if (pExec->withTbName) { + SMetaReader mr = {0}; + metaReaderInit(&mr, pTq->pVnode->pMeta, 0); + int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; + if (metaGetTableEntryByUid(&mr, uid) < 0) { + ASSERT(0); + } + char* tbName = strdup(mr.me.name); + taosArrayPush(rsp.blockTbName, &tbName); + metaReaderClear(&mr); + } + rsp.blockNum++; } // db subscribe @@ -563,6 +578,16 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { ASSERT(actualLen <= dataStrLen); taosArrayPush(rsp.blockDataLen, &actualLen); taosArrayPush(rsp.blockData, &buf); + if (pExec->withTbName) { + SMetaReader mr = {0}; + metaReaderInit(&mr, pTq->pVnode->pMeta, 0); + if (metaGetTableEntryByUid(&mr, block.info.uid) < 0) { + ASSERT(0); + } + char* tbName = strdup(mr.me.name); + taosArrayPush(rsp.blockTbName, &tbName); + metaReaderClear(&mr); + } SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader[workerId]->pSchemaWrapper); taosArrayPush(rsp.blockSchema, &pSW); @@ -614,6 +639,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { taosArrayDestroy(rsp.blockData); taosArrayDestroy(rsp.blockDataLen); taosArrayDestroyP(rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); + taosArrayDestroyP(rsp.blockTbName, (FDelete)taosMemoryFree); return 0; } From 72f9a1c4a4cff5ce41bf43316a2f47bb206cc494 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 12 May 2022 03:34:38 +0000 Subject: [PATCH 58/94] fix --- source/dnode/vnode/src/meta/metaQuery.c | 10 +++++++++- source/libs/tdb/src/db/tdbPCache.c | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 8d2a4ebcf3..7d0a87d79d 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -158,7 +158,9 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo skmDbKey.sver = sver; pKey = &skmDbKey; kLen = sizeof(skmDbKey); + metaRLock(pMeta); ret = tdbDbGet(pMeta->pSkmDb, pKey, kLen, &pVal, &vLen); + metaULock(pMeta); if (ret < 0) { return NULL; } @@ -181,6 +183,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo } struct SMCtbCursor { + SMeta *pMeta; TDBC *pCur; tb_uid_t suid; void *pKey; @@ -200,9 +203,13 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { return NULL; } + pCtbCur->pMeta = pMeta; pCtbCur->suid = uid; + metaRLock(pMeta); + ret = tdbDbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur, NULL); if (ret < 0) { + metaULock(pMeta); taosMemoryFree(pCtbCur); return NULL; } @@ -220,6 +227,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { if (pCtbCur) { + if (pCtbCur->pMeta) metaULock(pCtbCur->pMeta); if (pCtbCur->pCur) { tdbDbcClose(pCtbCur->pCur); @@ -269,7 +277,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { pSW = metaGetTableSchema(pMeta, quid, sver, 0); if (!pSW) return NULL; - + tdInitTSchemaBuilder(&sb, 0); for (int i = 0; i < pSW->nCols; i++) { pSchema = pSW->pSchema + i; diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index aa05687426..8574e071f2 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -242,7 +242,7 @@ static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) { int h; h = tdbPCachePageHash(&(pPage->pgid)); - for (ppPage = &(pCache->pgHash[h % pCache->nHash]); *ppPage != pPage; ppPage = &((*ppPage)->pHashNext)) + for (ppPage = &(pCache->pgHash[h % pCache->nHash]); (*ppPage) && *ppPage != pPage; ppPage = &((*ppPage)->pHashNext)) ; ASSERT(*ppPage == pPage); *ppPage = pPage->pHashNext; From 51094f3434549cdebf0fbfec3a1c2e9c0a7a814f Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 12 May 2022 11:53:47 +0800 Subject: [PATCH 59/94] fix: table would be null when destory commit handle --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 031d200a66..07a68d780a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -485,8 +485,10 @@ static void tsdbDestroyCommitIters(SCommitH *pCommith) { for (int i = 1; i < pCommith->niters; i++) { tSkipListDestroyIter(pCommith->iters[i].pIter); - tdFreeSchema(pCommith->iters[i].pTable->pSchema); - taosMemoryFree(pCommith->iters[i].pTable); + if (pCommith->iters[i].pTable) { + tdFreeSchema(pCommith->iters[i].pTable->pSchema); + taosMemoryFreeClear(pCommith->iters[i].pTable); + } } taosMemoryFree(pCommith->iters); From 67ae3c2efa8c254b8877a1b12e57a824072c0b9c Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 12 May 2022 11:55:58 +0800 Subject: [PATCH 60/94] fix: table would be null when destroy commit handle --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 031d200a66..07a68d780a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -485,8 +485,10 @@ static void tsdbDestroyCommitIters(SCommitH *pCommith) { for (int i = 1; i < pCommith->niters; i++) { tSkipListDestroyIter(pCommith->iters[i].pIter); - tdFreeSchema(pCommith->iters[i].pTable->pSchema); - taosMemoryFree(pCommith->iters[i].pTable); + if (pCommith->iters[i].pTable) { + tdFreeSchema(pCommith->iters[i].pTable->pSchema); + taosMemoryFreeClear(pCommith->iters[i].pTable); + } } taosMemoryFree(pCommith->iters); From 8b9a2c20f31224c7b9602e7910ebdafd18056a1b Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 12 May 2022 12:05:58 +0800 Subject: [PATCH 61/94] enh(sync): add error log, linux api error, %X --- source/dnode/vnode/src/vnd/vnodeSync.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 1260f9a3e7..f0f5338c4d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -72,6 +72,7 @@ int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg) { int32_t ret = 0; SMsgCb *pMsgCb = rpcHandle; if (pMsgCb->queueFps[SYNC_QUEUE] != NULL) { + pMsg->noResp = 1; tmsgSendReq(rpcHandle, pEpSet, pMsg); } else { vError("vnodeSendMsg queue is NULL, SYNC_QUEUE:%d", SYNC_QUEUE); From 77f4c3b5ab6ca568fc716bf3cb4fb9af048feebc Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 12 May 2022 13:30:05 +0800 Subject: [PATCH 62/94] fix: set commit table when move blk idx between different last file --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 42 ++++++++++++++++++++---- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 031d200a66..3521e9b1f5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -70,6 +70,7 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid); static void tsdbResetCommitFile(SCommitH *pCommith); static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid); static int tsdbCommitToTable(SCommitH *pCommith, int tid); +static bool tsdbCommitIsSameFile(SCommitH *pCommith, int bidx); static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx); static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable); static int tsdbComparKeyBlock(const void *arg1, const void *arg2); @@ -889,9 +890,11 @@ static int tsdbCommitToTable(SCommitH *pCommith, int tid) { } static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx) { - SReadH *pReadh = &pCommith->readh; - int nBlocks = pIdx->numOfBlocks; - int bidx = 0; + SReadH *pReadh = &pCommith->readh; + STsdb *pTsdb = TSDB_READ_REPO(pReadh); + STSchema *pTSchema = NULL; + int nBlocks = pIdx->numOfBlocks; + int bidx = 0; tsdbResetCommitTable(pCommith); @@ -901,30 +904,49 @@ static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx) { return -1; } + STable table = {.tid = pIdx->uid, .uid = pIdx->uid, .pSchema = NULL}; + pCommith->pTable = &table; + while (bidx < nBlocks) { + if (!pTSchema && !tsdbCommitIsSameFile(pCommith, bidx)) { + // Set commit table + pTSchema = metaGetTbTSchema(REPO_META(pTsdb), pIdx->uid, 0); // TODO: schema version + if (!pTSchema) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + table.pSchema = pTSchema; + if (tsdbSetCommitTable(pCommith, &table) < 0) { + taosMemoryFreeClear(pTSchema); + return -1; + } + } + if (tsdbMoveBlock(pCommith, bidx) < 0) { tsdbError("vgId:%d failed to move block into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno)); + taosMemoryFreeClear(pTSchema); return -1; } + ++bidx; } - STable table = {.tid = pIdx->uid, .uid = pIdx->uid, .pSchema = NULL}; - TSDB_COMMIT_TABLE(pCommith) = &table; - if (tsdbWriteBlockInfo(pCommith) < 0) { tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno)); + taosMemoryFreeClear(pTSchema); return -1; } + taosMemoryFreeClear(pTSchema); return 0; } static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable) { STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); + pCommith->pTable = pTable; if (tdInitDataCols(pCommith->pDataCols, pSchema) < 0) { @@ -1321,6 +1343,14 @@ static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) { return 0; } +static bool tsdbCommitIsSameFile(SCommitH *pCommith, int bidx) { + SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + if (pBlock->last) { + return pCommith->isLFileSame; + } + return pCommith->isDFileSame; +} + static int tsdbMoveBlock(SCommitH *pCommith, int bidx) { SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx; SDFile *pDFile; From c32a3400553cdfe65937be86e67c9173cbf404de Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 12 May 2022 06:13:59 +0000 Subject: [PATCH 63/94] fix memory leak --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 10 ++++++++++ source/libs/tdb/src/db/tdbBtree.c | 1 + 2 files changed, 11 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index d40a73eb67..1e8fbb48c7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -62,6 +62,16 @@ int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable) { void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable) { if (pMemTable) { taosHashCleanup(pMemTable->pHashIdx); + SSkipListIterator *pIter = tSkipListCreateIter(pMemTable->pSlIdx); + SSkipListNode *pNode = NULL; + STbData *pTbData = NULL; + for (;;) { + if (!tSkipListIterNext(pIter)) break; + pNode = tSkipListIterGet(pIter); + pTbData = (STbData *)pNode->pData; + tsdbFreeTbData(pTbData); + } + tSkipListDestroyIter(pIter); tSkipListDestroy(pMemTable->pSlIdx); taosMemoryFree(pMemTable); } diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index cf7dd50103..fffda68731 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -114,6 +114,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, tdb_cmpr_fn_t kcmpr, SB int tdbBtreeClose(SBTree *pBt) { if (pBt) { + tdbFree(pBt->pBuf); tdbOsFree(pBt); } return 0; From 1dbe0650e03d8316c185fea2fb2ef3cd531c9b2e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 12 May 2022 14:20:50 +0800 Subject: [PATCH 64/94] enh(wal): set errno code --- source/libs/wal/src/walRead.c | 13 ++++++++++--- source/libs/wal/src/walWrite.c | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 0cfe75bf33..7dfe1b8989 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -130,6 +130,7 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { } } + // code set inner if (walReadSeekFilePos(pRead, pRet->firstVer, ver) < 0) { return -1; } @@ -249,16 +250,22 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { // TODO: check wal life if (pRead->curVersion != ver) { if (walReadSeekVer(pRead, ver) < 0) { + terrno = TSDB_CODE_WAL_INVALID_VER; + wError("unexpected wal log version: % " PRId64 ", since seek error", ver); return -1; } } - if (!taosValidFile(pRead->pReadLogTFile)) { - return -1; - } + /*if (!taosValidFile(pRead->pReadLogTFile)) {*/ + /*return -1;*/ + /*}*/ code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead)); if (code != sizeof(SWalHead)) { + if (code < 0) + terrno = TAOS_SYSTEM_ERROR(errno); + else + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index da1c36dcc4..2e43997584 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -225,6 +225,7 @@ int walRoll(SWal *pWal) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + // terrno set inner code = walRollFileInfo(pWal); if (code != 0) { return -1; From 71e43677f74b7c6c217e979934b8c324985f661b Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 12 May 2022 14:38:56 +0800 Subject: [PATCH 65/94] feat(query): add csum function --- source/libs/function/inc/builtinsimpl.h | 3 ++ source/libs/function/src/builtins.c | 41 ++++++++++++++++++ source/libs/function/src/builtinsimpl.c | 57 ++++++++++++++++++++++++- 3 files changed, 99 insertions(+), 2 deletions(-) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index e3b7127efe..e1e30d37ea 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -94,6 +94,9 @@ bool stateFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t stateCountFunction(SqlFunctionCtx* pCtx); int32_t stateDurationFunction(SqlFunctionCtx* pCtx); +bool getCsumFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +int32_t csumFunction(SqlFunctionCtx* pCtx); + bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); #ifdef __cplusplus diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 07ad0f7d1f..c9136433e0 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -308,6 +308,37 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32 return TSDB_CODE_SUCCESS; } +static int32_t translateCsum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return TSDB_CODE_SUCCESS; + } + + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + if (QUERY_NODE_COLUMN != nodeType(pPara)) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "The input parameter of CSUM function can only be column"); + } + + uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + uint8_t resType; + if (!IS_NUMERIC_TYPE(colType)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } else { + if (IS_SIGNED_NUMERIC_TYPE(colType)) { + resType = TSDB_DATA_TYPE_BIGINT; + } else if (IS_UNSIGNED_NUMERIC_TYPE(colType)) { + resType = TSDB_DATA_TYPE_UBIGINT; + } else if (IS_FLOAT_TYPE(colType)) { + resType = TSDB_DATA_TYPE_DOUBLE; + } else { + ASSERT(0); + } + } + + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[resType].bytes, .type = resType}; + return TSDB_CODE_SUCCESS; +} + static int32_t translateLastRow(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // todo return TSDB_CODE_SUCCESS; @@ -742,6 +773,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = stateDurationFunction, .finalizeFunc = NULL }, + { + .name = "csum", + .type = FUNCTION_TYPE_CSUM, + .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateCsum, + .getEnvFunc = getCsumFuncEnv, + .initFunc = functionSetup, + .processFunc = csumFunction, + .finalizeFunc = NULL + }, { .name = "abs", .type = FUNCTION_TYPE_ABS, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index efc2992075..173dc9fd57 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2815,7 +2815,6 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - SColumnInfoData* pTsOutput = pCtx->pTsOutput; int32_t numOfElems = 0; SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; @@ -2853,7 +2852,6 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx) { TSKEY* tsList = (int64_t*)pInput->pPTS->pData; SColumnInfoData* pInputCol = pInput->pData[0]; - SColumnInfoData* pTsOutput = pCtx->pTsOutput; int32_t numOfElems = 0; SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; @@ -2893,3 +2891,58 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx) { return numOfElems; } + +bool getCsumFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { + pEnv->calcMemSize = sizeof(SSumRes); + return true; +} + +int32_t csumFunction(SqlFunctionCtx* pCtx) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SSumRes* pSumRes = GET_ROWCELL_INTERBUF(pResInfo); + + SInputColumnInfoData* pInput = &pCtx->input; + TSKEY* tsList = (int64_t*)pInput->pPTS->pData; + + SColumnInfoData* pInputCol = pInput->pData[0]; + SColumnInfoData* pTsOutput = pCtx->pTsOutput; + SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; + + int32_t numOfElems = 0; + int32_t type = pInputCol->info.type; + int32_t startOffset = pCtx->offset; + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + int32_t pos = startOffset + numOfElems; + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + //colDataAppendNULL(pOutput, i); + continue; + } + + char* data = colDataGetData(pInputCol, i); + if (IS_SIGNED_NUMERIC_TYPE(type)) { + int64_t v; + GET_TYPED_DATA(v, int64_t, type, data); + pSumRes->isum += v; + colDataAppend(pOutput, pos, (char *)&pSumRes->isum, false); + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + uint64_t v; + GET_TYPED_DATA(v, uint64_t, type, data); + pSumRes->usum += v; + colDataAppend(pOutput, pos, (char *)&pSumRes->usum, false); + } else if (IS_FLOAT_TYPE(type)) { + double v; + GET_TYPED_DATA(v, double, type, data); + pSumRes->dsum += v; + colDataAppend(pOutput, pos, (char *)&pSumRes->dsum, false); + } + + //TODO: remove this after pTsOutput is handled + if (pTsOutput != NULL) { + colDataAppendInt64(pTsOutput, pos, &tsList[i]); + } + + numOfElems++; + } + + return numOfElems; +} From d7bd682237e710f1a6d319150d113d82dc6d53dd Mon Sep 17 00:00:00 2001 From: "slzhou@taodata.com" Date: Thu, 12 May 2022 14:42:57 +0800 Subject: [PATCH 66/94] column has member hasNull --- include/libs/function/tudf.h | 2 ++ include/util/tcoding.h | 15 +++++++++++++++ source/common/src/tdatablock.c | 2 ++ source/libs/function/src/tudf.c | 2 ++ 4 files changed, 21 insertions(+) diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index bdccd29acf..8e156edcd2 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -89,6 +89,7 @@ typedef struct SUdfColumnData { typedef struct SUdfColumn { SUdfColumnMeta colMeta; + bool hasNull; SUdfColumnData colData; } SUdfColumn; @@ -232,6 +233,7 @@ static FORCE_INLINE void udfColDataSetNull(SUdfColumn* pColumn, int32_t row) { } else { udfColDataSetNull_f(pColumn, row); } + pColumn->hasNull = true; } static FORCE_INLINE int32_t udfColDataSet(SUdfColumn* pColumn, uint32_t currentRow, const char* pData, bool isNull) { diff --git a/include/util/tcoding.h b/include/util/tcoding.h index 3f00c79f46..74e64d5292 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -59,6 +59,21 @@ static FORCE_INLINE void *taosDecodeFixedI8(const void *buf, int8_t *value) { static FORCE_INLINE void *taosSkipFixedLen(const void *buf, size_t len) { return POINTER_SHIFT(buf, len); } +// --- Bool + +static FORCE_INLINE int32_t taosEncodeFixedBool(void **buf, bool value) { + if (buf != NULL) { + ((int8_t *)(*buf))[0] = value ? 1 : 0; + *buf = POINTER_SHIFT(*buf, sizeof(int8_t)); + } + return (int32_t)sizeof(int8_t); +} + +static FORCE_INLINE void *taosDecodeFixedBool(const void *buf, bool *value) { + *value = ((int8_t *)buf)[0] == 0 ? false : true; + return POINTER_SHIFT(buf, sizeof(int8_t)); +} + // ---- Fixed U16 static FORCE_INLINE int32_t taosEncodeFixedU16(void **buf, uint16_t value) { if (buf != NULL) { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index b58e4bd1dd..9053101938 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1311,6 +1311,7 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { tlen += taosEncodeFixedI16(buf, pColData->info.colId); tlen += taosEncodeFixedI16(buf, pColData->info.type); tlen += taosEncodeFixedI32(buf, pColData->info.bytes); + tlen += taosEncodeFixedBool(buf, pColData->hasNull); if (IS_VAR_DATA_TYPE(pColData->info.type)) { tlen += taosEncodeBinary(buf, pColData->varmeta.offset, sizeof(int32_t) * rows); @@ -1340,6 +1341,7 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { buf = taosDecodeFixedI16(buf, &data.info.colId); buf = taosDecodeFixedI16(buf, &data.info.type); buf = taosDecodeFixedI32(buf, &data.info.bytes); + buf = taosDecodeFixedBool(buf, &data.hasNull); if (IS_VAR_DATA_TYPE(data.info.type)) { buf = taosDecodeBinary(buf, (void**)&data.varmeta.offset, pBlock->info.rows * sizeof(int32_t)); diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index a577ea200f..9d90dca63e 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -695,6 +695,7 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo udfCol->colMeta.scale = col->info.scale; udfCol->colMeta.precision = col->info.precision; udfCol->colData.numOfRows = udfBlock->numOfRows; + udfCol->hasNull = col->hasNull; if (IS_VAR_DATA_TYPE(udfCol->colMeta.type)) { udfCol->colData.varLenCol.varOffsetsLen = sizeof(int32_t) * udfBlock->numOfRows; udfCol->colData.varLenCol.varOffsets = taosMemoryMalloc(udfCol->colData.varLenCol.varOffsetsLen); @@ -731,6 +732,7 @@ int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) { col->info.bytes = meta->bytes; col->info.scale = meta->scale; col->info.type = meta->type; + col->hasNull = udfCol->hasNull; SUdfColumnData *data = &udfCol->colData; if (!IS_VAR_DATA_TYPE(meta->type)) { From 06853043bd05ddccbd32b5fec71553aa73023fdf Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 12 May 2022 14:57:58 +0800 Subject: [PATCH 67/94] feat(tmq): add config msg.with.table.name --- example/src/tmq.c | 3 ++- include/common/tmsg.h | 1 + source/client/src/tmq.c | 39 ++++++++++++++++++++++++---------- source/dnode/vnode/src/tq/tq.c | 11 +++++++--- source/libs/wal/src/walRead.c | 8 ++----- 5 files changed, 41 insertions(+), 21 deletions(-) diff --git a/example/src/tmq.c b/example/src/tmq.c index 2ee91c254c..0b5f3be1b0 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -107,7 +107,7 @@ int32_t create_topic() { taos_free_result(pRes); /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/ - pRes = taos_query(pConn, "create topic topic_ctb_column with table as select ts, c1, c2, c3 from st1"); + pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; @@ -166,6 +166,7 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); /*tmq_conf_set(conf, "td.connect.db", "abc1");*/ + tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print, NULL); tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); assert(tmq); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d34892a278..7bb1d70421 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2371,6 +2371,7 @@ typedef struct { typedef struct { SMsgHead head; char subKey[TSDB_SUBSCRIBE_KEY_LEN]; + int8_t withTbName; int32_t epoch; uint64_t reqId; int64_t consumerId; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index b42f072e54..a6b8b842f9 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -57,16 +57,17 @@ struct tmq_topic_vgroup_list_t { }; struct tmq_conf_t { - char clientId[256]; - char groupId[TSDB_CGROUP_LEN]; - int8_t autoCommit; - int8_t resetOffset; - uint16_t port; - int32_t autoCommitInterval; - char* ip; - char* user; - char* pass; - char* db; + char clientId[256]; + char groupId[TSDB_CGROUP_LEN]; + int8_t autoCommit; + int8_t resetOffset; + int8_t withTbName; + uint16_t port; + int32_t autoCommitInterval; + char* ip; + char* user; + char* pass; + /*char* db;*/ tmq_commit_cb* commitCb; void* commitCbUserParam; }; @@ -75,6 +76,7 @@ struct tmq_t { // conf char groupId[TSDB_CGROUP_LEN]; char clientId[256]; + int8_t withTbName; int8_t autoCommit; int32_t autoCommitInterval; int32_t resetOffsetCfg; @@ -187,6 +189,7 @@ typedef struct { tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t)); + conf->withTbName = -1; conf->autoCommit = true; conf->autoCommitInterval = 5000; conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; @@ -240,6 +243,18 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } } + if (strcmp(key, "msg.with.table.name") == 0) { + if (strcmp(value, "true") == 0) { + conf->withTbName = 1; + } else if (strcmp(value, "false") == 0) { + conf->withTbName = 0; + } else if (strcmp(value, "none") == 0) { + conf->withTbName = -1; + } else { + return TMQ_CONF_INVALID; + } + } + if (strcmp(key, "td.connect.ip") == 0) { conf->ip = strdup(value); return TMQ_CONF_OK; @@ -257,7 +272,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value return TMQ_CONF_OK; } if (strcmp(key, "td.connect.db") == 0) { - conf->db = strdup(value); + /*conf->db = strdup(value);*/ return TMQ_CONF_OK; } @@ -485,6 +500,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { // set conf strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); + pTmq->withTbName = conf->withTbName; pTmq->autoCommit = conf->autoCommit; pTmq->autoCommitInterval = conf->autoCommitInterval; pTmq->commitCb = conf->commitCb; @@ -1104,6 +1120,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t waitTime, SMqClientTopic* pReq->subKey[tlen] = TMQ_SEPARATOR; strcpy(pReq->subKey + tlen + 1, pTopic->topicName); + pReq->withTbName = tmq->withTbName; pReq->waitTime = waitTime; pReq->consumerId = tmq->consumerId; pReq->epoch = tmq->epoch; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 29fabc0f9f..6f0c25fb91 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -427,13 +427,18 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqDataBlkRsp rsp = {0}; rsp.reqOffset = pReq->currentOffset; rsp.withSchema = pExec->withSchema; - rsp.withTbName = pExec->withTbName; rsp.blockData = taosArrayInit(0, sizeof(void*)); rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t)); rsp.blockSchema = taosArrayInit(0, sizeof(void*)); rsp.blockTbName = taosArrayInit(0, sizeof(void*)); + int8_t withTbName = pExec->withTbName; + if (pReq->withTbName != -1) { + withTbName = pReq->withTbName; + } + rsp.withTbName = withTbName; + while (1) { consumerEpoch = atomic_load_32(&pExec->epoch); if (consumerEpoch > reqEpoch) { @@ -538,7 +543,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { taosArrayPush(rsp.blockSchema, &pSW); } - if (pExec->withTbName) { + if (withTbName) { SMetaReader mr = {0}; metaReaderInit(&mr, pTq->pVnode->pMeta, 0); int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; @@ -578,7 +583,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { ASSERT(actualLen <= dataStrLen); taosArrayPush(rsp.blockDataLen, &actualLen); taosArrayPush(rsp.blockData, &buf); - if (pExec->withTbName) { + if (withTbName) { SMetaReader mr = {0}; metaReaderInit(&mr, pTq->pVnode->pMeta, 0); if (metaGetTableEntryByUid(&mr, block.info.uid) < 0) { diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 7dfe1b8989..64e6881cd0 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -155,9 +155,7 @@ int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead) { if (code < 0) return -1; } - if (!taosValidFile(pRead->pReadLogTFile)) { - return -1; - } + ASSERT(taosValidFile(pRead->pReadLogTFile) == true); code = taosReadFile(pRead->pReadLogTFile, pHead, sizeof(SWalHead)); if (code != sizeof(SWalHead)) { @@ -256,9 +254,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } } - /*if (!taosValidFile(pRead->pReadLogTFile)) {*/ - /*return -1;*/ - /*}*/ + ASSERT(taosValidFile(pRead->pReadLogTFile) == true); code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead)); if (code != sizeof(SWalHead)) { From 62b2ccedf13d245b1b1e5e44a0f7fce53d4fb6b8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 12 May 2022 07:09:27 +0000 Subject: [PATCH 68/94] return errno when table not exits --- include/common/tmsg.h | 3 ++- source/common/src/tmsg.c | 7 ++++--- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 11 +++++++++++ source/dnode/vnode/src/vnd/vnodeSvr.c | 9 ++++----- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d655b82a08..08d5765246 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -252,6 +252,7 @@ STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter); int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq* pReq, STSchema* pSchema); typedef struct { + int32_t code; int8_t hashMeta; int64_t uid; char* tblFName; @@ -271,7 +272,7 @@ typedef struct { int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); -void tFreeSSubmitRsp(SSubmitRsp *pRsp); +void tFreeSSubmitRsp(SSubmitRsp* pRsp); #define COL_SMA_ON ((int8_t)0x1) #define COL_IDX_ON ((int8_t)0x2) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index ebd81c7da3..021ee8455e 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4032,6 +4032,7 @@ int32_t tDecodeSVSubmitReq(SDecoder *pCoder, SVSubmitReq *pReq) { static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBlock) { if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI32(pEncoder, pBlock->code) < 0) return -1; if (tEncodeI8(pEncoder, pBlock->hashMeta) < 0) return -1; if (pBlock->hashMeta) { if (tEncodeI64(pEncoder, pBlock->uid) < 0) return -1; @@ -4047,10 +4048,11 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1; if (tDecodeI8(pDecoder, &pBlock->hashMeta) < 0) return -1; if (pBlock->hashMeta) { if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; - pBlock->tblFName= taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); + pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); if (NULL == pBlock->tblFName) return -1; if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; } @@ -4089,7 +4091,7 @@ int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) { if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1; } - tEndDecode(pDecoder); + tEndDecode(pDecoder); tDecoderClear(pDecoder); return 0; } @@ -4108,4 +4110,3 @@ void tFreeSSubmitRsp(SSubmitRsp *pRsp) { taosMemoryFree(pRsp); } - diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 1e8fbb48c7..037b099345 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -310,6 +310,17 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo TSKEY keyMax; SSubmitBlk *pBlkCopy; + // check if table exists + SMetaReader mr = {0}; + SMetaEntry me = {0}; + metaReaderInit(&mr, pTsdb->pVnode->pMeta, 0); + if (metaGetTableEntryByUid(&mr, pMsgIter->uid) < 0) { + metaReaderClear(&mr); + terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; + return -1; + } + metaReaderClear(&mr); + // create container is nedd tptr = taosHashGet(pMemTable->pHashIdx, &(pMsgIter->uid), sizeof(pMsgIter->uid)); if (tptr == NULL) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index fc2b6fe676..158e14f2ad 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -502,7 +502,7 @@ _exit: return 0; } -static int vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSubmitMsgIter *msgIter, const char *tags) { +static int vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSubmitMsgIter *msgIter, const char *tags) { SSubmitBlkIter blkIter = {0}; STSchema *pSchema = NULL; tb_uid_t suid = 0; @@ -544,7 +544,7 @@ static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char while (true) { if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; if (pBlock == NULL) break; - + vnodeDebugPrintSingleSubmitMsg(pMeta, pBlock, &msgIter, tags); } @@ -595,7 +595,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in if (metaCreateTable(pVnode->pMeta, version, &createTbReq) < 0) { if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { - pRsp->code = terrno; + submitBlkRsp.code = terrno; tDecoderClear(&decoder); goto _exit; } @@ -617,8 +617,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in } if (tsdbInsertTableData(pVnode->pTsdb, &msgIter, pBlock, &submitBlkRsp) < 0) { - pRsp->code = terrno; - goto _exit; + submitBlkRsp.code = terrno; } submitRsp.numOfRows += submitBlkRsp.numOfRows; From b69a26678c6fa8fee3d104da2fafd857fde3b506 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 12 May 2022 15:23:41 +0800 Subject: [PATCH 69/94] enh(sync): raft config change --- include/common/tmsgdef.h | 1 + source/libs/sync/inc/syncInt.h | 1 + source/libs/sync/inc/syncUtil.h | 5 +++ source/libs/sync/src/syncAppendEntries.c | 22 +++++++++--- source/libs/sync/src/syncCommit.c | 13 ++++++- source/libs/sync/src/syncMain.c | 44 ++++++++++++++++++++++-- source/libs/sync/src/syncUtil.c | 28 +++++++++++++++ 7 files changed, 107 insertions(+), 7 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 8e918c40f9..c7deaa7845 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -217,6 +217,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SYNC_UNKNOWN, "vnode-sync-unknown", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SYNC_COMMON_RESPONSE, "vnode-sync-common-response", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPLY_MSG, "vnode-sync-apply-msg", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CONFIG_CHANGE, "vnode-sync-config-change", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SYNC_VNODE, "vnode-sync-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index cb539e8379..8a21eea7b7 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -271,6 +271,7 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, S cJSON* syncNode2Json(const SSyncNode* pSyncNode); char* syncNode2Str(const SSyncNode* pSyncNode); char* syncNode2SimpleStr(const SSyncNode* pSyncNode); +void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg *newConfig); SSyncNode* syncNodeAcquire(int64_t rid); void syncNodeRelease(SSyncNode* pNode); diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 83f31c5dac..159af1610e 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -57,6 +57,11 @@ SyncIndex syncUtilMinIndex(SyncIndex a, SyncIndex b); SyncIndex syncUtilMaxIndex(SyncIndex a, SyncIndex b); void syncUtilMsgHtoN(void* msg); void syncUtilMsgNtoH(void* msg); +bool syncUtilIsData(tmsg_t msgType); +bool syncUtilUserPreCommit(tmsg_t msgType); +bool syncUtilUserCommit(tmsg_t msgType); +bool syncUtilUserRollback(tmsg_t msgType); + #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 6623ed1caa..aed19d042e 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -19,6 +19,7 @@ #include "syncRaftStore.h" #include "syncUtil.h" #include "syncVoteMgr.h" +#include "syncRaftCfg.h" // TLA+ Spec // HandleAppendEntriesRequest(i, j, m) == @@ -199,7 +200,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index); assert(pRollBackEntry != NULL); - if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) { + //if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) { + if (syncUtilUserRollback(pRollBackEntry->msgType)) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); @@ -227,7 +229,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + //if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pAppendEntry->index; cbMeta.isWeak = pAppendEntry->isWeak; @@ -258,7 +261,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + //if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pAppendEntry->index; cbMeta.isWeak = pAppendEntry->isWeak; @@ -320,7 +324,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); - if (ths->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + //if (ths->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + if (ths->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pEntry->index; cbMeta.isWeak = pEntry->isWeak; @@ -330,6 +335,15 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, cbMeta); } + // config change + if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) { + SSyncCfg newSyncCfg; + int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg); + ASSERT(ret == 0); + + syncNodeUpdateConfig(ths, &newSyncCfg); + } + rpcFreeCont(rpcMsg.pCont); syncEntryDestory(pEntry); } diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 5d6cbd2a58..620f0e9cd2 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -19,6 +19,7 @@ #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" +#include "syncRaftCfg.h" // \* Leader i advances its commitIndex. // \* This is done as a separate step from handling AppendEntries responses, @@ -101,7 +102,8 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); - if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + //if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + if (pSyncNode->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pEntry->index; cbMeta.isWeak = pEntry->isWeak; @@ -111,6 +113,15 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, cbMeta); } + // config change + if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) { + SSyncCfg newSyncCfg; + int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg); + ASSERT(ret == 0); + + syncNodeUpdateConfig(pSyncNode, &newSyncCfg); + } + rpcFreeCont(rpcMsg.pCont); syncEntryDestory(pEntry); } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 911d8384f0..da23d8415b 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -116,6 +116,15 @@ void syncStop(int64_t rid) { int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { int32_t ret = 0; + char *configChange = syncCfg2Str((SSyncCfg*)pSyncCfg); + SRpcMsg rpcMsg = {0}; + rpcMsg.msgType = TDMT_VND_SYNC_CONFIG_CHANGE; + rpcMsg.noResp = 1; + rpcMsg.contLen = strlen(configChange) + 1; + rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); + snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", configChange); + taosMemoryFree(configChange); + ret = syncPropose(rid, &rpcMsg, false); return ret; } @@ -849,6 +858,35 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { return s; } +void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg *newConfig) { + pSyncNode->pRaftCfg->cfg = *newConfig; + int32_t ret = raftCfgPersist(pSyncNode->pRaftCfg); + ASSERT(ret == 0); + + // init internal + pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex]; + syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId); + + // init peersNum, peers, peersId + pSyncNode->peersNum = pSyncNode->pRaftCfg->cfg.replicaNum - 1; + int j = 0; + for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { + if (i != pSyncNode->pRaftCfg->cfg.myIndex) { + pSyncNode->peersNodeInfo[j] = pSyncNode->pRaftCfg->cfg.nodeInfo[i]; + j++; + } + } + for (int i = 0; i < pSyncNode->peersNum; ++i) { + syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i]); + } + + // init replicaNum, replicasId + pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum; + for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { + syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); + } +} + SSyncNode* syncNodeAcquire(int64_t rid) { SSyncNode* pNode = taosAcquireRef(tsNodeRefId, rid); if (pNode == NULL) { @@ -1207,7 +1245,8 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + //if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pEntry->index; cbMeta.isWeak = pEntry->isWeak; @@ -1228,7 +1267,8 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + //if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pEntry->index; cbMeta.isWeak = pEntry->isWeak; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 3110c0b2a3..cf045a6926 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -212,4 +212,32 @@ void syncUtilMsgNtoH(void* msg) { SMsgHead* pHead = msg; pHead->contLen = ntohl(pHead->contLen); pHead->vgId = ntohl(pHead->vgId); +} + +bool syncUtilIsData(tmsg_t msgType) { + if (msgType == TDMT_VND_SYNC_NOOP || msgType == TDMT_VND_SYNC_CONFIG_CHANGE) { + return false; + } + return true; +} + +bool syncUtilUserPreCommit(tmsg_t msgType) { + if (msgType != TDMT_VND_SYNC_NOOP && msgType != TDMT_VND_SYNC_CONFIG_CHANGE) { + return true; + } + return false; +} + +bool syncUtilUserCommit(tmsg_t msgType) { + if (msgType != TDMT_VND_SYNC_NOOP && msgType != TDMT_VND_SYNC_CONFIG_CHANGE) { + return true; + } + return false; +} + +bool syncUtilUserRollback(tmsg_t msgType) { + if (msgType != TDMT_VND_SYNC_NOOP && msgType != TDMT_VND_SYNC_CONFIG_CHANGE) { + return true; + } + return false; } \ No newline at end of file From 4511f5828fb3f5e650acc5405366df9611722cfc Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 12 May 2022 15:34:08 +0800 Subject: [PATCH 70/94] test: add test case for tmq --- tests/system-test/7-tmq/basic5.py | 85 ++- tests/system-test/fulltest.sh | 4 + tests/test/c/tmqSim.c | 1026 +++++++++++++++-------------- 3 files changed, 608 insertions(+), 507 deletions(-) diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py index 99aa4e72aa..7ed9f7ebe7 100644 --- a/tests/system-test/7-tmq/basic5.py +++ b/tests/system-test/7-tmq/basic5.py @@ -61,7 +61,7 @@ class TDTestCase: tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum)) return - def insert_data(self,dbName,stbName,ctbNum,rowsPerTbl,startTs): + def insert_data(self,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs): tdLog.debug("start to insert data ............") tdSql.execute("use %s" %dbName) pre_insert = "insert into " @@ -72,13 +72,15 @@ class TDTestCase: sql += " %s_%d values "%(stbName,i) for j in range(rowsPerTbl): sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) - if (j > 0) and (j%2000 == 0): + if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): tdSql.execute(sql) - sql = "insert into %s_%d values " %(stbName,i) + if j < rowsPerTbl - 1: + sql = "insert into %s_%d values " %(stbName,i) + else: + sql = "insert into " #end sql if sql != pre_insert: - # print(sql) - print("sql:%s"%sql) + #print("insert sql:%s"%sql) tdSql.execute(sql) tdLog.debug("insert data ............ [OK]") return @@ -96,6 +98,7 @@ class TDTestCase: parameterDict["stbName"],\ parameterDict["ctbNum"],\ parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"],\ parameterDict["startTs"]) return @@ -117,13 +120,81 @@ class TDTestCase: 'vgroups': 1, \ 'stbName': 'stb', \ 'ctbNum': 10, \ - 'rowsPerTbl': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 10, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() + time.sleep(1) + # wait stb ready + while 1: + tdSql.query("show %s.stables"%parameterDict['dbName']) + if tdSql.getRows() == 1: + #if (self.queryRows == 1): + time.sleep(1) + break + + tdLog.info("create topics from super table") + topicFromStb = 'topic_stb_column' + topicFromCtb = 'topic_ctb_column' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName'])) + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName'])) + + tdSql.query("show topics") + tdSql.checkRows(2) + topic1 = tdSql.getData(0 , 0) + topic2 = tdSql.getData(1 , 0) + if topic1 != topicFromStb or topic1 != topicFromCtb: + tdLog.exit("topic error") + if topic2 != topicFromStb or topic2 != topicFromCtb: + tdLog.exit("topic error") + + tdLog.info("create consume info table and consume result table") + cdbName = 'cdb' + tdSql.query("create database %s"%cdbName) + tdSql.query("create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)") + tdSql.query("create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)") + + consumerId = 0 + expectmsgcnt = (parameterDict["rowsPerTbl"] / parameterDict["batchNum"] + 1) * parameterDict["ctbNum"] + topicList = topicFromStb + ifcheckdata = 0 + keyList = 'group.id:cgrp1, \ + enable.auto.commit:false, \ + auto.commit.interval.ms:6000, \ + auto.offset.reset:none' + sql = "insert into consumeinfo values " + sql += "(now, %d, '%s', '%s', %l64d, %d)"%(consumerId, topicList, keyList, expectmsgcnt, ifcheckdata) + tdSql.query(sql) + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s, -g %d, -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" + tdLog.info(shellCmd) + os.system(taosCmd) + # wait for data ready - prepareEnvThread.join() + prepareEnvThread.join() + + tdLog.info("check consume result") + while 1: + tdSql.query("select * from consumeresult") + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == 1: + #if (self.queryRows == 1): + time.sleep(1) + break + + tdSql.checkData(0 , 1, consumerId) + tdSql.checkData(0 , 2, expectmsgcnt) + tdSql.checkData(0 , 3, expectrowcnt) tdLog.printNoPrefix("======== test scenario 2: ") diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index a6b4408cdc..817f814873 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -51,3 +51,7 @@ python3 ./test.py -f 2-query/arcsin.py python3 ./test.py -f 2-query/arccos.py python3 ./test.py -f 2-query/arctan.py # python3 ./test.py -f 2-query/query_cols_tags_and_or.py + +python3 ./test.py -f 7-tmq/basic5.py + + diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 4a59d18d87..bc3aa091c3 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -1,500 +1,526 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "taos.h" -#include "taoserror.h" -#include "tlog.h" - -#define GREEN "\033[1;32m" -#define NC "\033[0m" -#define min(a, b) (((a) < (b)) ? (a) : (b)) - -#define MAX_SQL_STR_LEN (1024 * 1024) -#define MAX_ROW_STR_LEN (16 * 1024) -#define MAX_CONSUMER_THREAD_CNT (16) - -typedef struct { - TdThread thread; - int32_t consumerId; - - int32_t ifCheckData; - int64_t expectMsgCnt; - - int64_t consumeMsgCnt; - int64_t consumeRowCnt; - int32_t checkresult; - - char topicString[1024]; - char keyString[1024]; - - int32_t numOfTopic; - char topics[32][64]; - - int32_t numOfKey; - char key[32][64]; - char value[32][64]; - - tmq_t* tmq; - tmq_list_t* topicList; - -} SThreadInfo; - -typedef struct { - // input from argvs - char cdbName[32]; - char dbName[32]; - int32_t showMsgFlag; - int32_t showRowFlag; - int32_t consumeDelay; // unit s - int32_t numOfThread; - SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; -} SConfInfo; - -static SConfInfo g_stConfInfo; -TdFilePtr g_fp = NULL; - -// char* g_pRowValue = NULL; -// TdFilePtr g_fp = NULL; - -static void printHelp() { - char indent[10] = " "; - printf("Used to test the tmq feature with sim cases\n"); - - printf("%s%s\n", indent, "-c"); - printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); - printf("%s%s\n", indent, "-d"); - printf("%s%s%s\n", indent, indent, "The name of the database for cosumer, no default "); - printf("%s%s\n", indent, "-g"); - printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag); - printf("%s%s\n", indent, "-r"); - printf("%s%s%s%d\n", indent, indent, "showRowFlag, default is ", g_stConfInfo.showRowFlag); - printf("%s%s\n", indent, "-y"); - printf("%s%s%s%d\n", indent, indent, "consume delay, default is s", g_stConfInfo.consumeDelay); - exit(EXIT_SUCCESS); -} - -void initLogFile() { - // FILE *fp = fopen(g_stConfInfo.resultFileName, "a"); - char file[256]; - sprintf(file, "%s/../log/tmqlog.txt", configDir); - TdFilePtr pFile = taosOpenFile(file, TD_FILE_TEXT | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); - if (NULL == pFile) { - fprintf(stderr, "Failed to open %s for save result\n", "./tmqlog.txt"); - exit(-1); - } - g_fp = pFile; -} - -void saveConfigToLogFile() { - time_t tTime = taosGetTimestampSec(); - struct tm tm = *taosLocalTime(&tTime, NULL); - - taosFprintfFile(g_fp, "###################################################################\n"); - taosFprintfFile(g_fp, "# configDir: %s\n", configDir); - taosFprintfFile(g_fp, "# dbName: %s\n", g_stConfInfo.dbName); - taosFprintfFile(g_fp, "# cdbName: %s\n", g_stConfInfo.cdbName); - taosFprintfFile(g_fp, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); - taosFprintfFile(g_fp, "# showRowFlag: %d\n", g_stConfInfo.showRowFlag); - taosFprintfFile(g_fp, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); - taosFprintfFile(g_fp, "# numOfThread: %d\n", g_stConfInfo.numOfThread); - - for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { - taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); - taosFprintfFile(g_fp, " Topics: "); - for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) { - taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]); - } - taosFprintfFile(g_fp, "\n"); - taosFprintfFile(g_fp, " Key: "); - for (int k = 0; k < g_stConfInfo.stThreads[i].numOfKey; k++) { - taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]); - } - taosFprintfFile(g_fp, "\n"); - } - - taosFprintfFile(g_fp, "# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - taosFprintfFile(g_fp, "###################################################################\n"); -} - -void parseArgument(int32_t argc, char* argv[]) { - memset(&g_stConfInfo, 0, sizeof(SConfInfo)); - g_stConfInfo.showMsgFlag = 0; - g_stConfInfo.showRowFlag = 0; - g_stConfInfo.consumeDelay = 5; - - for (int32_t i = 1; i < argc; i++) { - if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { - printHelp(); - exit(0); - } else if (strcmp(argv[i], "-d") == 0) { - strcpy(g_stConfInfo.dbName, argv[++i]); - } else if (strcmp(argv[i], "-w") == 0) { - strcpy(g_stConfInfo.cdbName, argv[++i]); - } else if (strcmp(argv[i], "-c") == 0) { - strcpy(configDir, argv[++i]); - } else if (strcmp(argv[i], "-g") == 0) { - g_stConfInfo.showMsgFlag = atol(argv[++i]); - } else if (strcmp(argv[i], "-r") == 0) { - g_stConfInfo.showRowFlag = atol(argv[++i]); - } else if (strcmp(argv[i], "-y") == 0) { - g_stConfInfo.consumeDelay = atol(argv[++i]); - } else { - printf("%s unknow para: %s %s", GREEN, argv[++i], NC); - exit(-1); - } - } - - initLogFile(); - - taosFprintfFile(g_fp, "====parseArgument() success\n"); - -#if 1 - pPrint("%s configDir:%s %s", GREEN, configDir, NC); - pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC); - pPrint("%s cdbName:%s %s", GREEN, g_stConfInfo.cdbName, NC); - pPrint("%s consumeDelay:%d %s", GREEN, g_stConfInfo.consumeDelay, NC); - pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); - pPrint("%s showRowFlag:%d %s", GREEN, g_stConfInfo.showRowFlag, NC); -#endif -} - -void splitStr(char** arr, char* str, const char* del) { - char* s = strtok(str, del); - while (s != NULL) { - *arr++ = s; - s = strtok(NULL, del); - } -} - -void ltrim(char* str) { - if (str == NULL || *str == '\0') { - return; - } - int len = 0; - char* p = str; - while (*p != '\0' && isspace(*p)) { - ++p; - ++len; - } - memmove(str, p, strlen(str) - len + 1); - // return str; -} - -static int running = 1; -static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) { - char buf[1024]; - int32_t totalRows = 0; - - // printf("topic: %s\n", tmq_get_topic_name(msg)); - // printf("vg:%d\n", tmq_get_vgroup_id(msg)); - taosFprintfFile(g_fp, "msg index:%" PRId64 ", threadLable: %d\n", msgIndex, threadLable); - taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), tmq_get_vgroup_id(msg)); - - while (1) { - TAOS_ROW row = taos_fetch_row(msg); - if (row == NULL) break; - if (0 != g_stConfInfo.showRowFlag) { - TAOS_FIELD* fields = taos_fetch_fields(msg); - int32_t numOfFields = taos_field_count(msg); - taos_print_row(buf, row, fields, numOfFields); - taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf); - } - totalRows++; - } - - return totalRows; -} - -int queryDB(TAOS* taos, char* command) { - TAOS_RES* pRes = taos_query(taos, command); - int code = taos_errno(pRes); - // if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { - if (code != 0) { - pError("failed to reason:%s, sql: %s", tstrerror(code), command); - taos_free_result(pRes); - return -1; - } - taos_free_result(pRes); - return 0; -} - -static void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) { - printf("tmq_commit_cb_print() commit %d\n", resp); -} - -void build_consumer(SThreadInfo* pInfo) { - tmq_conf_t* conf = tmq_conf_new(); - - // tmq_conf_set(conf, "td.connect.ip", "localhost"); - // tmq_conf_set(conf, "td.connect.port", "6030"); - tmq_conf_set(conf, "td.connect.user", "root"); - tmq_conf_set(conf, "td.connect.pass", "taosdata"); - - tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); - - tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print, NULL); - - // tmq_conf_set(conf, "group.id", "cgrp1"); - for (int32_t i = 0; i < pInfo->numOfKey; i++) { - tmq_conf_set(conf, pInfo->key[i], pInfo->value[i]); - } - - // tmq_conf_set(conf, "client.id", "c-001"); - - // tmq_conf_set(conf, "enable.auto.commit", "true"); - // tmq_conf_set(conf, "enable.auto.commit", "false"); - - // tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); - - // tmq_conf_set(conf, "auto.offset.reset", "none"); - // tmq_conf_set(conf, "auto.offset.reset", "earliest"); - // tmq_conf_set(conf, "auto.offset.reset", "latest"); - - pInfo->tmq = tmq_consumer_new(conf, NULL, 0); - return; -} - -void build_topic_list(SThreadInfo* pInfo) { - pInfo->topicList = tmq_list_new(); - // tmq_list_append(topic_list, "test_stb_topic_1"); - for (int32_t i = 0; i < pInfo->numOfTopic; i++) { - tmq_list_append(pInfo->topicList, pInfo->topics[i]); - } - return; -} - -int32_t saveConsumeResult(SThreadInfo* pInfo) { - char sqlStr[1024] = {0}; - - TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int - sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", g_stConfInfo.cdbName, - pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult); - - TAOS_RES* pRes = taos_query(pConn, sqlStr); - if (taos_errno(pRes) != 0) { - printf("error in save consumeinfo, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - exit(-1); - } - - taos_free_result(pRes); - - return 0; -} - -void loop_consume(SThreadInfo* pInfo) { - tmq_resp_err_t err; - - int64_t totalMsgs = 0; - int64_t totalRows = 0; - - while (running) { - TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000); - if (tmqMsg) { - if (0 != g_stConfInfo.showMsgFlag) { - totalRows += msg_process(tmqMsg, totalMsgs, pInfo->consumerId); - } - - taos_free_result(tmqMsg); - - totalMsgs++; - - if (totalMsgs >= pInfo->expectMsgCnt) { - break; - } - } else { - break; - } - } - - pInfo->consumeMsgCnt = totalMsgs; - pInfo->consumeRowCnt = totalRows; - - taosFprintfFile(g_fp, "==== consumerId: %d, consumeMsgCnt: %" PRId64 ", consumeRowCnt: %" PRId64 "\n", - pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt); -} - -void* consumeThreadFunc(void* param) { - int32_t totalMsgs = 0; - - SThreadInfo* pInfo = (SThreadInfo*)param; - - build_consumer(pInfo); - build_topic_list(pInfo); - if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { - return NULL; - } - - tmq_resp_err_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); - if (err) { - printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); - exit(-1); - } - - loop_consume(pInfo); - - tmq_commit(pInfo->tmq, NULL, 0); - - err = tmq_unsubscribe(pInfo->tmq); - if (err) { - printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); - pInfo->consumeMsgCnt = -1; - return NULL; - } - - err = tmq_consumer_close(pInfo->tmq); - if (err) { - printf("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); - exit(-1); - } - pInfo->tmq = NULL; - - // save consume result into consumeresult table - saveConsumeResult(pInfo); - - return NULL; -} - -void parseConsumeInfo() { - char* token; - const char delim[2] = ","; - const char ch = ':'; - - for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { - token = strtok(g_stConfInfo.stThreads[i].topicString, delim); - while (token != NULL) { - // printf("%s\n", token ); - strcpy(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic], token); - ltrim(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic]); - // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); - g_stConfInfo.stThreads[i].numOfTopic++; - - token = strtok(NULL, delim); - } - - token = strtok(g_stConfInfo.stThreads[i].keyString, delim); - while (token != NULL) { - // printf("%s\n", token ); - { - char* pstr = token; - ltrim(pstr); - char* ret = strchr(pstr, ch); - memcpy(g_stConfInfo.stThreads[i].key[g_stConfInfo.stThreads[i].numOfKey], pstr, ret - pstr); - strcpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1); - // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], - // g_stConfInfo.value[g_stConfInfo.numOfKey]); - g_stConfInfo.stThreads[i].numOfKey++; - } - - token = strtok(NULL, delim); - } - } -} - -int32_t getConsumeInfo() { - char sqlStr[1024] = {0}; - - TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.cdbName); - TAOS_RES* pRes = taos_query(pConn, sqlStr); - if (taos_errno(pRes) != 0) { - printf("error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); - taosFprintfFile(g_fp, "error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); - taosCloseFile(&g_fp); - taos_free_result(pRes); - exit(-1); - } - - TAOS_ROW row = NULL; - int num_fields = taos_num_fields(pRes); - TAOS_FIELD* fields = taos_fetch_fields(pRes); - - // schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, - // ifcheckdata int - - int32_t numOfThread = 0; - while ((row = taos_fetch_row(pRes))) { - int32_t* lengths = taos_fetch_lengths(pRes); - - for (int i = 0; i < num_fields; ++i) { - if (row[i] == NULL || 0 == i) { - continue; - } - - if ((1 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { - g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t*)row[i]); - } else if ((2 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { - memcpy(g_stConfInfo.stThreads[numOfThread].topicString, row[i], lengths[i]); - } else if ((3 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { - memcpy(g_stConfInfo.stThreads[numOfThread].keyString, row[i], lengths[i]); - } else if ((4 == i) && (fields[i].type == TSDB_DATA_TYPE_BIGINT)) { - g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]); - } else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { - g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]); - } - } - numOfThread++; - } - g_stConfInfo.numOfThread = numOfThread; - - taos_free_result(pRes); - - parseConsumeInfo(); - - return 0; -} - -int main(int32_t argc, char* argv[]) { - parseArgument(argc, argv); - getConsumeInfo(); - saveConfigToLogFile(); - - TdThreadAttr thattr; - taosThreadAttrInit(&thattr); - taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - - // pthread_create one thread to consume - taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread); - for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) { - taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc, - (void*)(&(g_stConfInfo.stThreads[i]))); - } - - for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { - taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL); - } - - // printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); - - taosFprintfFile(g_fp, "==== close tmqlog ====\n"); - taosCloseFile(&g_fp); - - return 0; -} - +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "taos.h" +#include "taoserror.h" +#include "tlog.h" + +#define GREEN "\033[1;32m" +#define NC "\033[0m" +#define min(a, b) (((a) < (b)) ? (a) : (b)) + +#define MAX_SQL_STR_LEN (1024 * 1024) +#define MAX_ROW_STR_LEN (16 * 1024) +#define MAX_CONSUMER_THREAD_CNT (16) + +typedef struct { + TdThread thread; + int32_t consumerId; + + int32_t autoCommitIntervalMs; // 1000 ms + char autoCommit[8]; // true, false + char autoOffsetRest[16]; // none, earliest, latest + + int32_t ifCheckData; + int64_t expectMsgCnt; + + int64_t consumeMsgCnt; + int64_t consumeRowCnt; + int32_t checkresult; + + char topicString[1024]; + char keyString[1024]; + + int32_t numOfTopic; + char topics[32][64]; + + int32_t numOfKey; + char key[32][64]; + char value[32][64]; + + tmq_t* tmq; + tmq_list_t* topicList; + +} SThreadInfo; + +typedef struct { + // input from argvs + char cdbName[32]; + char dbName[32]; + int32_t showMsgFlag; + int32_t showRowFlag; + int32_t consumeDelay; // unit s + int32_t numOfThread; + SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; +} SConfInfo; + +static SConfInfo g_stConfInfo; +TdFilePtr g_fp = NULL; + +// char* g_pRowValue = NULL; +// TdFilePtr g_fp = NULL; + +static void printHelp() { + char indent[10] = " "; + printf("Used to test the tmq feature with sim cases\n"); + + printf("%s%s\n", indent, "-c"); + printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); + printf("%s%s\n", indent, "-d"); + printf("%s%s%s\n", indent, indent, "The name of the database for cosumer, no default "); + printf("%s%s\n", indent, "-g"); + printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%d\n", indent, indent, "showRowFlag, default is ", g_stConfInfo.showRowFlag); + printf("%s%s\n", indent, "-y"); + printf("%s%s%s%d\n", indent, indent, "consume delay, default is s", g_stConfInfo.consumeDelay); + exit(EXIT_SUCCESS); +} + +void initLogFile() { + // FILE *fp = fopen(g_stConfInfo.resultFileName, "a"); + char file[256]; + sprintf(file, "%s/../log/tmqlog.txt", configDir); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_TEXT | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + if (NULL == pFile) { + fprintf(stderr, "Failed to open %s for save result\n", "./tmqlog.txt"); + exit(-1); + } + g_fp = pFile; +} + +void saveConfigToLogFile() { + time_t tTime = taosGetTimestampSec(); + struct tm tm = *taosLocalTime(&tTime, NULL); + + taosFprintfFile(g_fp, "###################################################################\n"); + taosFprintfFile(g_fp, "# configDir: %s\n", configDir); + taosFprintfFile(g_fp, "# dbName: %s\n", g_stConfInfo.dbName); + taosFprintfFile(g_fp, "# cdbName: %s\n", g_stConfInfo.cdbName); + taosFprintfFile(g_fp, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); + taosFprintfFile(g_fp, "# showRowFlag: %d\n", g_stConfInfo.showRowFlag); + taosFprintfFile(g_fp, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); + taosFprintfFile(g_fp, "# numOfThread: %d\n", g_stConfInfo.numOfThread); + + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); + taosFprintfFile(g_fp, " auto commit: %s\n", g_stConfInfo.stThreads[i].autoCommit); + taosFprintfFile(g_fp, " auto commit interval ms: %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs); + taosFprintfFile(g_fp, " auto offset rest: %s\n", g_stConfInfo.stThreads[i].autoOffsetRest); + taosFprintfFile(g_fp, " Topics: "); + for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) { + taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]); + } + taosFprintfFile(g_fp, "\n"); + taosFprintfFile(g_fp, " Key: "); + for (int k = 0; k < g_stConfInfo.stThreads[i].numOfKey; k++) { + taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]); + } + taosFprintfFile(g_fp, "\n"); + } + + taosFprintfFile(g_fp, "# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + taosFprintfFile(g_fp, "###################################################################\n"); +} + +void parseArgument(int32_t argc, char* argv[]) { + memset(&g_stConfInfo, 0, sizeof(SConfInfo)); + g_stConfInfo.showMsgFlag = 0; + g_stConfInfo.showRowFlag = 0; + g_stConfInfo.consumeDelay = 5; + + for (int32_t i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-d") == 0) { + strcpy(g_stConfInfo.dbName, argv[++i]); + } else if (strcmp(argv[i], "-w") == 0) { + strcpy(g_stConfInfo.cdbName, argv[++i]); + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-g") == 0) { + g_stConfInfo.showMsgFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + g_stConfInfo.showRowFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-y") == 0) { + g_stConfInfo.consumeDelay = atol(argv[++i]); + } else { + printf("%s unknow para: %s %s", GREEN, argv[++i], NC); + exit(-1); + } + } + + initLogFile(); + + taosFprintfFile(g_fp, "====parseArgument() success\n"); + +#if 1 + pPrint("%s configDir:%s %s", GREEN, configDir, NC); + pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC); + pPrint("%s cdbName:%s %s", GREEN, g_stConfInfo.cdbName, NC); + pPrint("%s consumeDelay:%d %s", GREEN, g_stConfInfo.consumeDelay, NC); + pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); + pPrint("%s showRowFlag:%d %s", GREEN, g_stConfInfo.showRowFlag, NC); +#endif +} + +void splitStr(char** arr, char* str, const char* del) { + char* s = strtok(str, del); + while (s != NULL) { + *arr++ = s; + s = strtok(NULL, del); + } +} + +void ltrim(char* str) { + if (str == NULL || *str == '\0') { + return; + } + int len = 0; + char* p = str; + while (*p != '\0' && isspace(*p)) { + ++p; + ++len; + } + memmove(str, p, strlen(str) - len + 1); + // return str; +} + +static int running = 1; +static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) { + char buf[1024]; + int32_t totalRows = 0; + + // printf("topic: %s\n", tmq_get_topic_name(msg)); + // printf("vg:%d\n", tmq_get_vgroup_id(msg)); + taosFprintfFile(g_fp, "msg index:%" PRId64 ", threadLable: %d\n", msgIndex, threadLable); + taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), tmq_get_vgroup_id(msg)); + + while (1) { + TAOS_ROW row = taos_fetch_row(msg); + if (row == NULL) break; + if (0 != g_stConfInfo.showRowFlag) { + TAOS_FIELD* fields = taos_fetch_fields(msg); + int32_t numOfFields = taos_field_count(msg); + taos_print_row(buf, row, fields, numOfFields); + taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf); + } + totalRows++; + } + + return totalRows; +} + +int queryDB(TAOS* taos, char* command) { + TAOS_RES* pRes = taos_query(taos, command); + int code = taos_errno(pRes); + // if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { + if (code != 0) { + pError("failed to reason:%s, sql: %s", tstrerror(code), command); + taos_free_result(pRes); + return -1; + } + taos_free_result(pRes); + return 0; +} + +static void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) { + printf("tmq_commit_cb_print() commit %d\n", resp); +} + +void build_consumer(SThreadInfo* pInfo) { + tmq_conf_t* conf = tmq_conf_new(); + + // tmq_conf_set(conf, "td.connect.ip", "localhost"); + // tmq_conf_set(conf, "td.connect.port", "6030"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + + //tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); + + tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print, NULL); + + // tmq_conf_set(conf, "group.id", "cgrp1"); + for (int32_t i = 0; i < pInfo->numOfKey; i++) { + tmq_conf_set(conf, pInfo->key[i], pInfo->value[i]); + } + + // tmq_conf_set(conf, "client.id", "c-001"); + + // tmq_conf_set(conf, "enable.auto.commit", "true"); + // tmq_conf_set(conf, "enable.auto.commit", "false"); + + // tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); + + // tmq_conf_set(conf, "auto.offset.reset", "none"); + // tmq_conf_set(conf, "auto.offset.reset", "earliest"); + // tmq_conf_set(conf, "auto.offset.reset", "latest"); + + pInfo->tmq = tmq_consumer_new(conf, NULL, 0); + + tmq_conf_destroy(conf); + + return; +} + +void build_topic_list(SThreadInfo* pInfo) { + pInfo->topicList = tmq_list_new(); + // tmq_list_append(topic_list, "test_stb_topic_1"); + for (int32_t i = 0; i < pInfo->numOfTopic; i++) { + tmq_list_append(pInfo->topicList, pInfo->topics[i]); + } + return; +} + +int32_t saveConsumeResult(SThreadInfo* pInfo) { + char sqlStr[1024] = {0}; + + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int + sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", g_stConfInfo.cdbName, + pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult); + + TAOS_RES* pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("error in save consumeinfo, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + exit(-1); + } + + taos_free_result(pRes); + + return 0; +} + +void loop_consume(SThreadInfo* pInfo) { + tmq_resp_err_t err; + + int64_t totalMsgs = 0; + int64_t totalRows = 0; + + while (running) { + TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000); + if (tmqMsg) { + if (0 != g_stConfInfo.showMsgFlag) { + totalRows += msg_process(tmqMsg, totalMsgs, pInfo->consumerId); + } + + taos_free_result(tmqMsg); + + totalMsgs++; + + if (totalMsgs >= pInfo->expectMsgCnt) { + taosFprintfFile(g_fp, "==== totalMsgs >= pInfo->expectMsgCnt, so break\n"); + break; + } + } else { + taosFprintfFile(g_fp, "==== delay over time, so break\n"); + break; + } + } + + pInfo->consumeMsgCnt = totalMsgs; + pInfo->consumeRowCnt = totalRows; + + taosFprintfFile(g_fp, "==== consumerId: %d, consumeMsgCnt: %" PRId64 ", consumeRowCnt: %" PRId64 "\n", + pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt); +} + +void* consumeThreadFunc(void* param) { + int32_t totalMsgs = 0; + + SThreadInfo* pInfo = (SThreadInfo*)param; + + build_consumer(pInfo); + build_topic_list(pInfo); + if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { + return NULL; + } + + tmq_resp_err_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); + if (err) { + printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); + exit(-1); + } + + tmq_list_destroy(pInfo->topicList); + pInfo->topicList = NULL; + + loop_consume(pInfo); + + tmq_commit(pInfo->tmq, NULL, 0); + + err = tmq_unsubscribe(pInfo->tmq); + if (err) { + printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); + pInfo->consumeMsgCnt = -1; + return NULL; + } + + err = tmq_consumer_close(pInfo->tmq); + if (err) { + printf("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); + exit(-1); + } + pInfo->tmq = NULL; + + // save consume result into consumeresult table + saveConsumeResult(pInfo); + + return NULL; +} + +void parseConsumeInfo() { + char* token; + const char delim[2] = ","; + const char ch = ':'; + + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + token = strtok(g_stConfInfo.stThreads[i].topicString, delim); + while (token != NULL) { + // printf("%s\n", token ); + strcpy(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic], token); + ltrim(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic]); + // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); + g_stConfInfo.stThreads[i].numOfTopic++; + + token = strtok(NULL, delim); + } + + token = strtok(g_stConfInfo.stThreads[i].keyString, delim); + while (token != NULL) { + // printf("%s\n", token ); + { + char* pstr = token; + ltrim(pstr); + char* ret = strchr(pstr, ch); + memcpy(g_stConfInfo.stThreads[i].key[g_stConfInfo.stThreads[i].numOfKey], pstr, ret - pstr); + strcpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1); + // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], + // g_stConfInfo.value[g_stConfInfo.numOfKey]); + g_stConfInfo.stThreads[i].numOfKey++; + } + + token = strtok(NULL, delim); + } + } +} + +int32_t getConsumeInfo() { + char sqlStr[1024] = {0}; + + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.cdbName); + TAOS_RES* pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); + taosFprintfFile(g_fp, "error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); + taosCloseFile(&g_fp); + taos_free_result(pRes); + exit(-1); + } + + TAOS_ROW row = NULL; + int num_fields = taos_num_fields(pRes); + TAOS_FIELD* fields = taos_fetch_fields(pRes); + + // schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, + // ifcheckdata int + + int32_t numOfThread = 0; + while ((row = taos_fetch_row(pRes))) { + int32_t* lengths = taos_fetch_lengths(pRes); + + // set default value + g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000; + memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true")); + memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast")); + + for (int i = 0; i < num_fields; ++i) { + if (row[i] == NULL || 0 == i) { + continue; + } + + if ((1 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { + g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t*)row[i]); + } else if ((2 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { + memcpy(g_stConfInfo.stThreads[numOfThread].topicString, row[i], lengths[i]); + } else if ((3 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { + memcpy(g_stConfInfo.stThreads[numOfThread].keyString, row[i], lengths[i]); + } else if ((4 == i) && (fields[i].type == TSDB_DATA_TYPE_BIGINT)) { + g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]); + } else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { + g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]); + } else if ((6 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { + memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, row[i], lengths[i]); + } else if ((7 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { + g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = *((int32_t*)row[i]); + } else if ((8 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { + memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, row[i], lengths[i]); + } + } + numOfThread++; + } + g_stConfInfo.numOfThread = numOfThread; + + taos_free_result(pRes); + + parseConsumeInfo(); + + return 0; +} + +int main(int32_t argc, char* argv[]) { + parseArgument(argc, argv); + getConsumeInfo(); + saveConfigToLogFile(); + + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + + // pthread_create one thread to consume + taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread); + for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) { + taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc, + (void*)(&(g_stConfInfo.stThreads[i]))); + } + + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL); + } + + // printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); + + taosFprintfFile(g_fp, "==== close tmqlog ====\n"); + taosCloseFile(&g_fp); + + return 0; +} + From 3309dd71db1f3152e64eb83095b1a6a737484147 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 12 May 2022 15:36:30 +0800 Subject: [PATCH 71/94] test: add test case for tmq --- tests/system-test/7-tmq/basic5.py | 143 +++++++++++++++++++----------- 1 file changed, 91 insertions(+), 52 deletions(-) diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py index 7ed9f7ebe7..267b6ff5d8 100644 --- a/tests/system-test/7-tmq/basic5.py +++ b/tests/system-test/7-tmq/basic5.py @@ -13,14 +13,12 @@ from util.dnodes import * class TDTestCase: hostname = socket.gethostname() - rpcDebugFlagVal = '143' - clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} - clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal - - updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} - updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal - - print ("===================: ", updatecfgDict) + #rpcDebugFlagVal = '143' + #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #print ("===================: ", updatecfgDict) def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") @@ -43,27 +41,35 @@ class TDTestCase: break return buildPath - def create_tables(self,dbName,vgroups,stbName,ctbNum,rowsPerTbl): - tdSql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) - tdSql.execute("use %s" %dbName) - tdSql.execute("create table %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) + def newcur(self,cfg,host,port): + user = "root" + password = "taosdata" + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): + tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("use %s" %dbName) + tsql.execute("create table %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" sql = pre_create #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) for i in range(ctbNum): sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1) if (i > 0) and (i%100 == 0): - tdSql.execute(sql) + tsql.execute(sql) sql = pre_create if sql != pre_create: - tdSql.execute(sql) + tsql.execute(sql) tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum)) return - def insert_data(self,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs): + def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs): tdLog.debug("start to insert data ............") - tdSql.execute("use %s" %dbName) + tsql.execute("use %s" %dbName) pre_insert = "insert into " sql = pre_insert @@ -73,7 +79,7 @@ class TDTestCase: for j in range(rowsPerTbl): sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): - tdSql.execute(sql) + tsql.execute(sql) if j < rowsPerTbl - 1: sql = "insert into %s_%d values " %(stbName,i) else: @@ -81,25 +87,29 @@ class TDTestCase: #end sql if sql != pre_insert: #print("insert sql:%s"%sql) - tdSql.execute(sql) + tsql.execute(sql) tdLog.debug("insert data ............ [OK]") return def prepareEnv(self, **parameterDict): print ("input parameters:") print (parameterDict) - self.create_tables(parameterDict["dbName"],\ + # create new connector for my thread + tsql=self.newcur(parameterDict['cfg'], 'localhost', 6030) + self.create_tables(tsql,\ + parameterDict["dbName"],\ parameterDict["vgroups"],\ parameterDict["stbName"],\ parameterDict["ctbNum"],\ parameterDict["rowsPerTbl"]) - self.insert_data(parameterDict["dbName"],\ - parameterDict["stbName"],\ - parameterDict["ctbNum"],\ - parameterDict["rowsPerTbl"],\ - parameterDict["batchNum"],\ - parameterDict["startTs"]) + self.insert_data(tsql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"],\ + parameterDict["startTs"]) return def run(self): @@ -116,24 +126,29 @@ class TDTestCase: tdLog.printNoPrefix("======== test scenario 1: ") tdLog.info("step 1: create database, stb, ctb and insert data") # create and start thread - parameterDict = {'dbName': 'db', \ + parameterDict = {'cfg': '', \ + 'dbName': 'db', \ 'vgroups': 1, \ 'stbName': 'stb', \ 'ctbNum': 10, \ - 'rowsPerTbl': 10000, \ + 'rowsPerTbl': 10000, \ 'batchNum': 10, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() - time.sleep(1) + time.sleep(2) # wait stb ready while 1: - tdSql.query("show %s.stables"%parameterDict['dbName']) - if tdSql.getRows() == 1: - #if (self.queryRows == 1): - time.sleep(1) + #tdSql.query("show %s.stables"%parameterDict['dbName']) + tdSql.query("show db.stables") + #print (self.queryResult) + #print (tdSql.getRows()) + if tdSql.getRows() == 1: break + else: + time.sleep(1) tdLog.info("create topics from super table") topicFromStb = 'topic_stb_column' @@ -141,57 +156,81 @@ class TDTestCase: tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName'])) - + + time.sleep(1) tdSql.query("show topics") - tdSql.checkRows(2) + print ("======================================") + #print (self.queryResult) + #tdSql.checkRows(2) topic1 = tdSql.getData(0 , 0) topic2 = tdSql.getData(1 , 0) - if topic1 != topicFromStb or topic1 != topicFromCtb: - tdLog.exit("topic error") - if topic2 != topicFromStb or topic2 != topicFromCtb: - tdLog.exit("topic error") + print (topic1) + print (topic2) + + print (topicFromStb) + print (topicFromCtb) + #tdLog.info("show topics: %s, %s"%topic1, topic2) + #if topic1 != topicFromStb or topic1 != topicFromCtb: + # tdLog.exit("topic error1") + #if topic2 != topicFromStb or topic2 != topicFromCtb: + # tdLog.exit("topic error2") tdLog.info("create consume info table and consume result table") - cdbName = 'cdb' - tdSql.query("create database %s"%cdbName) + cdbName = parameterDict["dbName"] + #tdSql.query("create database %s"%cdbName) + #tdSql.query("use %s"%cdbName) tdSql.query("create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)") tdSql.query("create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)") consumerId = 0 - expectmsgcnt = (parameterDict["rowsPerTbl"] / parameterDict["batchNum"] + 1) * parameterDict["ctbNum"] + expectmsgcnt = (parameterDict["rowsPerTbl"] / parameterDict["batchNum"] ) * parameterDict["ctbNum"] + expectmsgcnt1 = expectmsgcnt + parameterDict["ctbNum"] topicList = topicFromStb ifcheckdata = 0 - keyList = 'group.id:cgrp1, \ - enable.auto.commit:false, \ - auto.commit.interval.ms:6000, \ - auto.offset.reset:none' + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' sql = "insert into consumeinfo values " - sql += "(now, %d, '%s', '%s', %l64d, %d)"%(consumerId, topicList, keyList, expectmsgcnt, ifcheckdata) + sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectmsgcnt1, ifcheckdata) tdSql.query(sql) + tdLog.info("check stb if there are data") + while 1: + tdSql.query("select count(*) from %s"%parameterDict["stbName"]) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + countOfStb = tdSql.getData(0, 0) + if countOfStb != 0: + tdLog.info("count from stb: %d"%countOfStb) + break + else: + time.sleep(1) + tdLog.info("start consume processor") pollDelay = 5 showMsg = 1 showRow = 1 shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath - shellCmd += " -y %d -d %s, -g %d, -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) - os.system(taosCmd) + os.system(shellCmd) # wait for data ready prepareEnvThread.join() - tdLog.info("check consume result") + tdLog.info("insert process end, and start to check consume result") while 1: tdSql.query("select * from consumeresult") #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) if tdSql.getRows() == 1: - #if (self.queryRows == 1): - time.sleep(1) break - + else: + time.sleep(5) + + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + tdSql.checkData(0 , 1, consumerId) tdSql.checkData(0 , 2, expectmsgcnt) tdSql.checkData(0 , 3, expectrowcnt) From c95d3cdb36e18e5a33cb18f8e0bce460793b5a33 Mon Sep 17 00:00:00 2001 From: "slzhou@taodata.com" Date: Thu, 12 May 2022 15:43:32 +0800 Subject: [PATCH 72/94] feature(udf):error follows tsdb_code standard --- include/libs/function/tudf.h | 9 --------- include/util/taoserror.h | 10 +++++++++- source/libs/function/src/tudf.c | 14 +++++++------- source/libs/function/src/udfd.c | 2 +- source/util/src/terror.c | 8 ++++++++ 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index 8e156edcd2..b5c38e14f4 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -39,15 +39,6 @@ extern "C" { //====================================================================================== //begin API to taosd and qworker -enum { - UDFC_CODE_STOPPING = -1, - UDFC_CODE_PIPE_READ_ERR = -2, - UDFC_CODE_CONNECT_PIPE_ERR = -3, - UDFC_CODE_LOAD_UDF_FAILURE = -4, - UDFC_CODE_INVALID_STATE = -5, - UDFC_CODE_NO_PIPE = -6, -}; - typedef void *UdfcFuncHandle; /** diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 52b2f0c670..58cdd2cab4 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -645,7 +645,15 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FUNC_FUNTION_PARA_NUM TAOS_DEF_ERROR_CODE(0, 0x2801) #define TSDB_CODE_FUNC_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2802) #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE TAOS_DEF_ERROR_CODE(0, 0x2803) -#define TSDB_CODE_FUNC_INVALID_FUNTION TAOS_DEF_ERROR_CODE(0, 0x2604) +#define TSDB_CODE_FUNC_INVALID_FUNTION TAOS_DEF_ERROR_CODE(0, 0x2804) + +//udf +#define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901) +#define TSDB_CODE_UDF_PIPE_READ_ERR TAOS_DEF_ERROR_CODE(0, 0x2902) +#define TSDB_CODE_UDF_PIPE_CONNECT_ERR TAOS_DEF_ERROR_CODE(0, 0x2903) +#define TSDB_CODE_UDF_PIPE_NO_PIPE TAOS_DEF_ERROR_CODE(0, 0x2904) +#define TSDB_CODE_UDF_LOAD_UDF_FAILURE TAOS_DEF_ERROR_CODE(0, 0x2905) +#define TSDB_CODE_UDF_INVALID_STATE TAOS_DEF_ERROR_CODE(0, 0x2906) #define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000) #define TSDB_CODE_SML_INVALID_PRECISION_TYPE TAOS_DEF_ERROR_CODE(0, 0x3001) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 9d90dca63e..dfa7fac15a 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -931,7 +931,7 @@ void udfcUvHandleError(SClientUvConn *conn) { while (!QUEUE_EMPTY(&conn->taskQueue)) { QUEUE* h = QUEUE_HEAD(&conn->taskQueue); SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue); - task->errCode = UDFC_CODE_PIPE_READ_ERR; + task->errCode = TSDB_CODE_UDF_PIPE_READ_ERR; QUEUE_REMOVE(&task->connTaskQueue); QUEUE_REMOVE(&task->procTaskQueue); uv_sem_post(&task->taskSem); @@ -1119,7 +1119,7 @@ void cleanUpUvTasks(SUdfdProxy *udfc) { QUEUE_REMOVE(h); SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue); if (udfc->gUdfcState == UDFC_STATE_STOPPING) { - task->errCode = UDFC_CODE_STOPPING; + task->errCode = TSDB_CODE_UDF_STOPPING; } uv_sem_post(&task->taskSem); } @@ -1129,7 +1129,7 @@ void cleanUpUvTasks(SUdfdProxy *udfc) { QUEUE_REMOVE(h); SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, procTaskQueue); if (udfc->gUdfcState == UDFC_STATE_STOPPING) { - task->errCode = UDFC_CODE_STOPPING; + task->errCode = TSDB_CODE_UDF_STOPPING; } uv_sem_post(&task->taskSem); } @@ -1213,7 +1213,7 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) { int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) { fnInfo("udfc setup udf. udfName: %s", udfName); if (gUdfdProxy.gUdfcState != UDFC_STATE_READY) { - return UDFC_CODE_INVALID_STATE; + return TSDB_CODE_UDF_INVALID_STATE; } SClientUdfTask *task = taosMemoryCalloc(1,sizeof(SClientUdfTask)); task->errCode = 0; @@ -1227,7 +1227,7 @@ int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) { int32_t errCode = udfcRunUdfUvTask(task, UV_TASK_CONNECT); if (errCode != 0) { fnError("failed to connect to pipe. udfName: %s, pipe: %s", udfName, (&gUdfdProxy)->udfdPipeName); - return UDFC_CODE_CONNECT_PIPE_ERR; + return TSDB_CODE_UDF_PIPE_CONNECT_ERR; } udfcRunUdfUvTask(task, UV_TASK_REQ_RSP); @@ -1254,7 +1254,7 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf SClientUdfUvSession *session = (SClientUdfUvSession *) handle; if (session->udfUvPipe == NULL) { fnError("No pipe to udfd"); - return UDFC_CODE_NO_PIPE; + return TSDB_CODE_UDF_PIPE_NO_PIPE; } SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask)); task->errCode = 0; @@ -1374,7 +1374,7 @@ int32_t teardownUdf(UdfcFuncHandle handle) { SClientUdfUvSession *session = (SClientUdfUvSession *) handle; if (session->udfUvPipe == NULL) { fnError("pipe to udfd does not exist"); - return UDFC_CODE_NO_PIPE; + return TSDB_CODE_UDF_PIPE_NO_PIPE; } SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask)); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 0ad4674cfa..f006695f14 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -102,7 +102,7 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { int err = uv_dlopen(udf->path, &udf->lib); if (err != 0) { fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); - return UDFC_CODE_LOAD_UDF_FAILURE; + return TSDB_CODE_UDF_LOAD_UDF_FAILURE; } char initFuncName[TSDB_FUNC_NAME_LEN+5] = {0}; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index e32ecfc695..9676e4e1f9 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -454,6 +454,14 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PERMISSION_DENIED, "Permission denied") //planner TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR, "planner internal error") +//udf +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_READ_ERR, "udf pipe read error") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_CONNECT_ERR, "udf pipe connect error") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NO_PIPE, "udf no pipe") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE, "udf load failure") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE, "udf invalid state") + //schemaless TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type") TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp precision type") From bc4cd5ef3da28effd8ceec8b025292a8ba2970d1 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Thu, 12 May 2022 16:21:31 +0800 Subject: [PATCH 73/94] test: add ts filter testcases --- .../2-query/query_cols_tags_and_or.py | 118 ++++++++++-------- 1 file changed, 69 insertions(+), 49 deletions(-) diff --git a/tests/system-test/2-query/query_cols_tags_and_or.py b/tests/system-test/2-query/query_cols_tags_and_or.py index a62960cf43..77e91aa983 100644 --- a/tests/system-test/2-query/query_cols_tags_and_or.py +++ b/tests/system-test/2-query/query_cols_tags_and_or.py @@ -93,104 +93,124 @@ class TDTestCase: res = tdSql.query(query_sql.replace('*', 'last(*)'), True) return int(res[0][-4]) - def queryTsCol(self, tb_name): + def queryTsCol(self, tb_name, check_elm=None): + select_elm = "*" if check_elm is None else check_elm # ts and ts - query_sql = f'select * from {tb_name} where ts > "2021-01-11 12:00:00" or ts < "2021-01-13 12:00:00"' + query_sql = f'select {select_elm} from {tb_name} where ts > "2021-01-11 12:00:00" or ts < "2021-01-13 12:00:00"' tdSql.query(query_sql) tdSql.checkRows(11) - tdSql.checkEqual(self.queryLastC10(query_sql), 11) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False - query_sql = f'select * from {tb_name} where ts >= "2021-01-11 12:00:00" and ts <= "2021-01-13 12:00:00"' + query_sql = f'select {select_elm} from {tb_name} where ts >= "2021-01-11 12:00:00" and ts <= "2021-01-13 12:00:00"' tdSql.query(query_sql) - # tdSql.checkRows(2) - # tdSql.checkEqual(self.queryLastC10(query_sql), 6) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 6) if select_elm == "*" else False ## ts or and tinyint col - query_sql = f'select * from {tb_name} where ts > "2021-01-11 12:00:00" or c1 = 2' - tdSql.error(query_sql) + query_sql = f'select {select_elm} from {tb_name} where ts > "2021-01-11 12:00:00" or c1 = 2' + tdSql.query(query_sql) + tdSql.checkRows(7) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False - query_sql = f'select * from {tb_name} where ts <= "2021-01-11 12:00:00" and c1 != 2' + query_sql = f'select {select_elm} from {tb_name} where ts <= "2021-01-11 12:00:00" and c1 != 2' tdSql.query(query_sql) tdSql.checkRows(4) - tdSql.checkEqual(self.queryLastC10(query_sql), 5) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) if select_elm == "*" else False ## ts or and smallint col - query_sql = f'select * from {tb_name} where ts <> "2021-01-11 12:00:00" or c2 = 10' - tdSql.error(query_sql) + query_sql = f'select {select_elm} from {tb_name} where ts <> "2021-01-11 12:00:00" or c2 = 10' + tdSql.query(query_sql) + tdSql.checkRows(10) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False - query_sql = f'select * from {tb_name} where ts <= "2021-01-11 12:00:00" and c2 <= 1' + + query_sql = f'select {select_elm} from {tb_name} where ts <= "2021-01-11 12:00:00" and c2 <= 1' tdSql.query(query_sql) tdSql.checkRows(1) - tdSql.checkEqual(self.queryLastC10(query_sql), 1) + tdSql.checkEqual(self.queryLastC10(query_sql), 1) if select_elm == "*" else False ## ts or and int col - query_sql = f'select * from {tb_name} where ts >= "2021-01-11 12:00:00" or c3 = 4' - tdSql.error(query_sql) + query_sql = f'select {select_elm} from {tb_name} where ts >= "2021-01-11 12:00:00" or c3 = 4' + tdSql.query(query_sql) + tdSql.checkRows(8) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False - query_sql = f'select * from {tb_name} where ts < "2021-01-11 12:00:00" and c3 = 4' + query_sql = f'select {select_elm} from {tb_name} where ts < "2021-01-11 12:00:00" and c3 = 4' tdSql.query(query_sql) tdSql.checkRows(1) - tdSql.checkEqual(self.queryLastC10(query_sql), 4) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) if select_elm == "*" else False ## ts or and big col - query_sql = f'select * from {tb_name} where ts is Null or c4 = 5' - tdSql.error(query_sql) - - query_sql = f'select * from {tb_name} where ts is not Null and c4 = 2' + query_sql = f'select {select_elm} from {tb_name} where ts is Null or c4 = 5' tdSql.query(query_sql) tdSql.checkRows(1) - tdSql.checkEqual(self.queryLastC10(query_sql), 3) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) if select_elm == "*" else False + + query_sql = f'select {select_elm} from {tb_name} where ts is not Null and c4 = 2' + tdSql.query(query_sql) + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 3) if select_elm == "*" else False ## ts or and float col - query_sql = f'select * from {tb_name} where ts between "2021-01-17 12:00:00" and "2021-01-23 12:00:00" or c5 = 6.6' - tdSql.error(query_sql) + query_sql = f'select {select_elm} from {tb_name} where ts between "2021-01-17 12:00:00" and "2021-01-23 12:00:00" or c5 = 6.6' + tdSql.query(query_sql) + tdSql.checkRows(5) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False - query_sql = f'select * from {tb_name} where ts < "2021-01-11 12:00:00" and c5 = 1.1' + query_sql = f'select {select_elm} from {tb_name} where ts < "2021-01-11 12:00:00" and c5 = 1.1' tdSql.query(query_sql) tdSql.checkRows(4) - tdSql.checkEqual(self.queryLastC10(query_sql), 4) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) if select_elm == "*" else False ## ts or and double col - query_sql = f'select * from {tb_name} where ts between "2021-01-17 12:00:00" and "2021-01-23 12:00:00" or c6 = 7.7' - tdSql.error(query_sql) + query_sql = f'select {select_elm} from {tb_name} where ts between "2021-01-17 12:00:00" and "2021-01-23 12:00:00" or c6 = 7.7' + tdSql.query(query_sql) + tdSql.checkRows(5) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False - query_sql = f'select * from {tb_name} where ts < "2021-01-11 12:00:00" and c6 = 1.1' + query_sql = f'select {select_elm} from {tb_name} where ts < "2021-01-11 12:00:00" and c6 = 1.1' tdSql.query(query_sql) tdSql.checkRows(4) - tdSql.checkEqual(self.queryLastC10(query_sql), 4) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) if select_elm == "*" else False ## ts or and binary col - query_sql = f'select * from {tb_name} where ts < "2021-01-11 12:00:00" or c7 like "binary_"' - tdSql.error(query_sql) - - query_sql = f'select * from {tb_name} where ts <= "2021-01-11 12:00:00" and c7 in ("binary")' + query_sql = f'select {select_elm} from {tb_name} where ts < "2021-01-11 12:00:00" or c7 like "binary_"' tdSql.query(query_sql) tdSql.checkRows(5) - tdSql.checkEqual(self.queryLastC10(query_sql), 5) + tdSql.checkEqual(self.queryLastC10(query_sql), 8) if select_elm == "*" else False + + query_sql = f'select {select_elm} from {tb_name} where ts <= "2021-01-11 12:00:00" and c7 in ("binary")' + tdSql.query(query_sql) + tdSql.checkRows(5) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) if select_elm == "*" else False ## ts or and nchar col - query_sql = f'select * from {tb_name} where ts < "2021-01-11 12:00:00" or c8 like "nchar%"' - tdSql.error(query_sql) + query_sql = f'select {select_elm} from {tb_name} where ts < "2021-01-11 12:00:00" or c8 like "nchar%"' + tdSql.query(query_sql) + tdSql.checkRows(10) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) if select_elm == "*" else False - query_sql = f'select * from {tb_name} where ts >= "2021-01-11 12:00:00" and c8 is Null' + query_sql = f'select {select_elm} from {tb_name} where ts >= "2021-01-11 12:00:00" and c8 is Null' tdSql.query(query_sql) tdSql.checkRows(1) - tdSql.checkEqual(self.queryLastC10(query_sql), 11) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False ## ts or and bool col - query_sql = f'select * from {tb_name} where ts < "2021-01-11 12:00:00" or c9=false' - tdSql.error(query_sql) + query_sql = f'select {select_elm} from {tb_name} where ts < "2021-01-11 12:00:00" or c9=false' + tdSql.query(query_sql) + tdSql.checkRows(6) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False - query_sql = f'select * from {tb_name} where ts >= "2021-01-11 12:00:00" and c9=true' + query_sql = f'select {select_elm} from {tb_name} where ts >= "2021-01-11 12:00:00" and c9=true' tdSql.query(query_sql) tdSql.checkRows(5) - tdSql.checkEqual(self.queryLastC10(query_sql), 9) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) if select_elm == "*" else False ## multi cols - query_sql = f'select * from {tb_name} where ts > "2021-01-03 12:00:00" and c1 != 2 and c2 >= 2 and c3 <> 4 and c4 < 4 and c5 > 1 and c6 >= 1.1 and c7 is not Null and c8 = "nchar" and c9=false' + query_sql = f'select {select_elm} from {tb_name} where ts > "2021-01-03 12:00:00" and c1 != 2 and c2 >= 2 and c3 <> 4 and c4 < 4 and c5 > 1 and c6 >= 1.1 and c7 is not Null and c8 = "nchar" and c9=false' tdSql.query(query_sql) tdSql.checkRows(1) - tdSql.checkEqual(self.queryLastC10(query_sql), 10) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) if select_elm == "*" else False def queryTsTag(self, tb_name): ## ts and tinyint col @@ -2029,12 +2049,12 @@ class TDTestCase: tb_name = self.initStb() self.queryFullTagType(tb_name) - def checkTbTsCol(self): + def checkTbTsCol(self, check_elm): ''' Ordinary table ts and col check ''' tb_name = self.initTb() - self.queryTsCol(tb_name) + self.queryTsCol(tb_name, check_elm) def checkStbTsTol(self): tb_name = self.initStb() @@ -2112,8 +2132,8 @@ class TDTestCase: for check_elm in [None, column_name]: self.checkTbColTypeOperator(check_elm) self.checkStbColTypeOperator(check_elm) + self.checkTbTsCol(check_elm) # self.checkStbTagTypeOperator() - # self.checkTbTsCol() # self.checkStbTsTol() # self.checkStbTsTag() # self.checkStbTsColTag() From 265f2a858252c69f7a258e7ec018c25c767347b1 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 12 May 2022 16:36:08 +0800 Subject: [PATCH 74/94] test: add get rows api in test frame --- tests/pytest/util/sql.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 19a5d3ecc1..adbab04e07 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -150,6 +150,9 @@ class TDSql: raise Exception(repr(e)) return (self.queryRows, timeout) + def getRows(self): + return self.queryRows + def checkRows(self, expectRows): if self.queryRows == expectRows: tdLog.info("sql:%s, queryRows:%d == expect:%d" % (self.sql, self.queryRows, expectRows)) From 89271550133c6e0ae50feade6c686ffa200b8033 Mon Sep 17 00:00:00 2001 From: "slzhou@taodata.com" Date: Thu, 12 May 2022 16:49:31 +0800 Subject: [PATCH 75/94] feat:add udf dedicated errors --- include/util/taoserror.h | 1 + source/common/src/tglobal.c | 4 +- source/libs/function/src/tudf.c | 2 +- source/libs/function/src/udfd.c | 338 ++++++++++++++++--------------- source/libs/function/test/udf2.c | 6 + source/util/src/terror.c | 1 + 6 files changed, 190 insertions(+), 162 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 58cdd2cab4..238600160a 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -654,6 +654,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_UDF_PIPE_NO_PIPE TAOS_DEF_ERROR_CODE(0, 0x2904) #define TSDB_CODE_UDF_LOAD_UDF_FAILURE TAOS_DEF_ERROR_CODE(0, 0x2905) #define TSDB_CODE_UDF_INVALID_STATE TAOS_DEF_ERROR_CODE(0, 0x2906) +#define TSDB_CODE_UDF_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2907) #define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000) #define TSDB_CODE_SML_INVALID_PRECISION_TYPE TAOS_DEF_ERROR_CODE(0, 0x3001) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index f73a982110..42c8e18f3e 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -444,7 +444,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, 1) != 0) return -1; - if (cfgAddBool(pCfg, "startUdfd", tsStartUdfd, 0) != 0) return -1; + if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1; return 0; } @@ -585,7 +585,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32; tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32; - tsStartUdfd = cfgGetItem(pCfg, "startUdfd")->bval; + tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; if (tsQueryBufferSize >= 0) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index dfa7fac15a..8e96a2a063 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1497,7 +1497,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { taosArrayDestroy(tempBlock.pDataBlock); taosMemoryFree(newState.buf); - return TSDB_CODE_SUCCESS; + return udfCode; } int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index f006695f14..34681dc6cd 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -140,6 +140,182 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { return 0; } +void udfdProcessSetupRequest(SUvUdfWork* uvUdf, SUdfRequest* request) { + // TODO: tracable id from client. connect, setup, call, teardown + fnInfo("%" PRId64 " setup request. udf name: %s", request->seqNum, request->setup.udfName); + SUdfSetupRequest *setup = &request->setup; + int32_t code = TSDB_CODE_SUCCESS; + SUdf *udf = NULL; + uv_mutex_lock(&global.udfsMutex); + SUdf **udfInHash = taosHashGet(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName)); + if (udfInHash) { + ++(*udfInHash)->refCount; + udf = *udfInHash; + uv_mutex_unlock(&global.udfsMutex); + } else { + SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf)); + udfNew->refCount = 1; + udfNew->state = UDF_STATE_INIT; + + uv_mutex_init(&udfNew->lock); + uv_cond_init(&udfNew->condReady); + udf = udfNew; + taosHashPut(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName), &udfNew, sizeof(&udfNew)); + uv_mutex_unlock(&global.udfsMutex); + } + + uv_mutex_lock(&udf->lock); + if (udf->state == UDF_STATE_INIT) { + udf->state = UDF_STATE_LOADING; + code = udfdLoadUdf(setup->udfName, udf); + if (udf->initFunc) { + udf->initFunc(); + } + udf->state = UDF_STATE_READY; + uv_cond_broadcast(&udf->condReady); + uv_mutex_unlock(&udf->lock); + } else { + while (udf->state != UDF_STATE_READY) { + uv_cond_wait(&udf->condReady, &udf->lock); + } + uv_mutex_unlock(&udf->lock); + } + SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle)); + handle->udf = udf; + + SUdfResponse rsp; + rsp.seqNum = request->seqNum; + rsp.type = request->type; + rsp.code = code; + rsp.setupRsp.udfHandle = (int64_t)(handle); + rsp.setupRsp.outputType = udf->outputType; + rsp.setupRsp.outputLen = udf->outputLen; + rsp.setupRsp.bufSize = udf->bufSize; + + int32_t len = encodeUdfResponse(NULL, &rsp); + rsp.msgLen = len; + void *bufBegin = taosMemoryMalloc(len); + void *buf = bufBegin; + encodeUdfResponse(&buf, &rsp); + + uvUdf->output = uv_buf_init(bufBegin, len); + + taosMemoryFree(uvUdf->input.base); + return; +} + +void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { + SUdfCallRequest *call = &request->call; + fnDebug("%" PRId64 "call request. call type %d, handle: %" PRIx64, request->seqNum, call->callType, + call->udfHandle); + SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(call->udfHandle); + SUdf *udf = handle->udf; + SUdfResponse response = {0}; + SUdfResponse *rsp = &response; + SUdfCallResponse *subRsp = &rsp->callRsp; + + int32_t code = TSDB_CODE_SUCCESS; + switch(call->callType) { + case TSDB_UDF_CALL_SCALA_PROC: { + SUdfColumn output = {0}; + + SUdfDataBlock input = {0}; + convertDataBlockToUdfDataBlock(&call->block, &input); + code = udf->scalarProcFunc(&input, &output); + + convertUdfColumnToDataBlock(&output, &response.callRsp.resultData); + freeUdfColumn(&output); + break; + } + case TSDB_UDF_CALL_AGG_INIT: { + SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), + .bufLen= udf->bufSize, + .numOfResult = 0}; + udf->aggStartFunc(&outBuf); + subRsp->resultBuf = outBuf; + break; + } + case TSDB_UDF_CALL_AGG_PROC: { + SUdfDataBlock input = {0}; + convertDataBlockToUdfDataBlock(&call->block, &input); + SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), + .bufLen= udf->bufSize, + .numOfResult = 0}; + code = udf->aggProcFunc(&input, &call->interBuf, &outBuf); + subRsp->resultBuf = outBuf; + + break; + } + case TSDB_UDF_CALL_AGG_FIN: { + SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), + .bufLen= udf->bufSize, + .numOfResult = 0}; + code = udf->aggFinishFunc(&call->interBuf, &outBuf); + subRsp->resultBuf = outBuf; + break; + } + default: + break; + } + + rsp->seqNum = request->seqNum; + rsp->type = request->type; + rsp->code = code; + subRsp->callType = call->callType; + + int32_t len = encodeUdfResponse(NULL, rsp); + rsp->msgLen = len; + void *bufBegin = taosMemoryMalloc(len); + void *buf = bufBegin; + encodeUdfResponse(&buf, rsp); + uvUdf->output = uv_buf_init(bufBegin, len); + + taosMemoryFree(uvUdf->input.base); + return; +} + +void udfdProcessTeardownRequest(SUvUdfWork* uvUdf, SUdfRequest* request) { + SUdfTeardownRequest *teardown = &request->teardown; + fnInfo("teardown. %" PRId64 "handle:%" PRIx64, request->seqNum, teardown->udfHandle); + SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(teardown->udfHandle); + SUdf *udf = handle->udf; + bool unloadUdf = false; + int32_t code = TSDB_CODE_SUCCESS; + + uv_mutex_lock(&global.udfsMutex); + udf->refCount--; + if (udf->refCount == 0) { + unloadUdf = true; + taosHashRemove(global.udfsHash, udf->name, strlen(udf->name)); + } + uv_mutex_unlock(&global.udfsMutex); + if (unloadUdf) { + uv_cond_destroy(&udf->condReady); + uv_mutex_destroy(&udf->lock); + if (udf->destroyFunc) { + (udf->destroyFunc)(); + } + uv_dlclose(&udf->lib); + taosMemoryFree(udf); + } + taosMemoryFree(handle); + + SUdfResponse response; + SUdfResponse *rsp = &response; + rsp->seqNum = request->seqNum; + rsp->type = request->type; + rsp->code = code; + int32_t len = encodeUdfResponse(NULL, rsp); + rsp->msgLen = len; + void *bufBegin = taosMemoryMalloc(len); + void *buf = bufBegin; + encodeUdfResponse(&buf, rsp); + uvUdf->output = uv_buf_init(bufBegin, len); + + taosMemoryFree(uvUdf->input.base); + return; +} + void udfdProcessRequest(uv_work_t *req) { SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data); SUdfRequest request = {0}; @@ -147,172 +323,16 @@ void udfdProcessRequest(uv_work_t *req) { switch (request.type) { case UDF_TASK_SETUP: { - // TODO: tracable id from client. connect, setup, call, teardown - fnInfo("%" PRId64 " setup request. udf name: %s", request.seqNum, request.setup.udfName); - SUdfSetupRequest *setup = &request.setup; - - SUdf *udf = NULL; - uv_mutex_lock(&global.udfsMutex); - SUdf **udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, strlen(request.setup.udfName)); - if (udfInHash) { - ++(*udfInHash)->refCount; - udf = *udfInHash; - uv_mutex_unlock(&global.udfsMutex); - } else { - SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf)); - udfNew->refCount = 1; - udfNew->state = UDF_STATE_INIT; - - uv_mutex_init(&udfNew->lock); - uv_cond_init(&udfNew->condReady); - udf = udfNew; - taosHashPut(global.udfsHash, request.setup.udfName, strlen(request.setup.udfName), &udfNew, sizeof(&udfNew)); - uv_mutex_unlock(&global.udfsMutex); - } - - uv_mutex_lock(&udf->lock); - if (udf->state == UDF_STATE_INIT) { - udf->state = UDF_STATE_LOADING; - udfdLoadUdf(setup->udfName, udf); - if (udf->initFunc) { - udf->initFunc(); - } - udf->state = UDF_STATE_READY; - uv_cond_broadcast(&udf->condReady); - uv_mutex_unlock(&udf->lock); - } else { - while (udf->state != UDF_STATE_READY) { - uv_cond_wait(&udf->condReady, &udf->lock); - } - uv_mutex_unlock(&udf->lock); - } - SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle)); - handle->udf = udf; - SUdfResponse rsp; - rsp.seqNum = request.seqNum; - rsp.type = request.type; - rsp.code = 0; - rsp.setupRsp.udfHandle = (int64_t)(handle); - rsp.setupRsp.outputType = udf->outputType; - rsp.setupRsp.outputLen = udf->outputLen; - rsp.setupRsp.bufSize = udf->bufSize; - int32_t len = encodeUdfResponse(NULL, &rsp); - rsp.msgLen = len; - void *bufBegin = taosMemoryMalloc(len); - void *buf = bufBegin; - encodeUdfResponse(&buf, &rsp); - - uvUdf->output = uv_buf_init(bufBegin, len); - - taosMemoryFree(uvUdf->input.base); + udfdProcessSetupRequest(uvUdf, &request); break; } case UDF_TASK_CALL: { - SUdfCallRequest *call = &request.call; - fnDebug("%" PRId64 "call request. call type %d, handle: %" PRIx64, request.seqNum, call->callType, - call->udfHandle); - SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(call->udfHandle); - SUdf *udf = handle->udf; - SUdfResponse response = {0}; - SUdfResponse *rsp = &response; - SUdfCallResponse *subRsp = &rsp->callRsp; - - switch(call->callType) { - case TSDB_UDF_CALL_SCALA_PROC: { - SUdfColumn output = {0}; - - SUdfDataBlock input = {0}; - convertDataBlockToUdfDataBlock(&call->block, &input); - udf->scalarProcFunc(&input, &output); - - convertUdfColumnToDataBlock(&output, &response.callRsp.resultData); - freeUdfColumn(&output); - break; - } - case TSDB_UDF_CALL_AGG_INIT: { - SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), - .bufLen= udf->bufSize, - .numOfResult = 0}; - udf->aggStartFunc(&outBuf); - subRsp->resultBuf = outBuf; - break; - } - case TSDB_UDF_CALL_AGG_PROC: { - SUdfDataBlock input = {0}; - convertDataBlockToUdfDataBlock(&call->block, &input); - SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), - .bufLen= udf->bufSize, - .numOfResult = 0}; - udf->aggProcFunc(&input, &call->interBuf, &outBuf); - subRsp->resultBuf = outBuf; - - break; - } - case TSDB_UDF_CALL_AGG_FIN: { - SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), - .bufLen= udf->bufSize, - .numOfResult = 0}; - udf->aggFinishFunc(&call->interBuf, &outBuf); - subRsp->resultBuf = outBuf; - break; - } - default: - break; - } - - rsp->seqNum = request.seqNum; - rsp->type = request.type; - rsp->code = 0; - subRsp->callType = call->callType; - - int32_t len = encodeUdfResponse(NULL, rsp); - rsp->msgLen = len; - void *bufBegin = taosMemoryMalloc(len); - void *buf = bufBegin; - encodeUdfResponse(&buf, rsp); - uvUdf->output = uv_buf_init(bufBegin, len); - - taosMemoryFree(uvUdf->input.base); + udfdProcessCallRequest(uvUdf, &request); break; } case UDF_TASK_TEARDOWN: { - SUdfTeardownRequest *teardown = &request.teardown; - fnInfo("teardown. %" PRId64 "handle:%" PRIx64, request.seqNum, teardown->udfHandle) SUdfcFuncHandle *handle = - (SUdfcFuncHandle *)(teardown->udfHandle); - SUdf *udf = handle->udf; - bool unloadUdf = false; - uv_mutex_lock(&global.udfsMutex); - udf->refCount--; - if (udf->refCount == 0) { - unloadUdf = true; - taosHashRemove(global.udfsHash, udf->name, strlen(udf->name)); - } - uv_mutex_unlock(&global.udfsMutex); - if (unloadUdf) { - uv_cond_destroy(&udf->condReady); - uv_mutex_destroy(&udf->lock); - if (udf->destroyFunc) { - (udf->destroyFunc)(); - } - uv_dlclose(&udf->lib); - taosMemoryFree(udf); - } - taosMemoryFree(handle); - - SUdfResponse response; - SUdfResponse *rsp = &response; - rsp->seqNum = request.seqNum; - rsp->type = request.type; - rsp->code = 0; - int32_t len = encodeUdfResponse(NULL, rsp); - rsp->msgLen = len; - void *bufBegin = taosMemoryMalloc(len); - void *buf = bufBegin; - encodeUdfResponse(&buf, rsp); - uvUdf->output = uv_buf_init(bufBegin, len); - - taosMemoryFree(uvUdf->input.base); + udfdProcessTeardownRequest(uvUdf, &request); break; } default: { diff --git a/source/libs/function/test/udf2.c b/source/libs/function/test/udf2.c index be485bc905..b3b60f93a4 100644 --- a/source/libs/function/test/udf2.c +++ b/source/libs/function/test/udf2.c @@ -27,6 +27,12 @@ int32_t udf2_start(SUdfInterBuf *buf) { int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { int64_t sumSquares = *(int64_t*)interBuf->buf; int8_t numOutput = 0; + for (int32_t i = 0; i < block->numOfCols; ++i) { + SUdfColumn* col = block->udfCols[i]; + if (col->colMeta.type != TSDB_DATA_TYPE_INT) { + return TSDB_CODE_UDF_INVALID_INPUT; + } + } for (int32_t i = 0; i < block->numOfCols; ++i) { for (int32_t j = 0; j < block->numOfRows; ++j) { SUdfColumn* col = block->udfCols[i]; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 9676e4e1f9..a1bc37e6cd 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -461,6 +461,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_CONNECT_ERR, "udf pipe connect erro TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NO_PIPE, "udf no pipe") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE, "udf load failure") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE, "udf invalid state") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_INPUT, "udf invalid function input") //schemaless TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type") From 2218213134003ff021647c5cb664f3a004fe4acc Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 12 May 2022 17:01:33 +0800 Subject: [PATCH 76/94] enh(wal): add log --- source/libs/wal/src/walRead.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 64e6881cd0..b04dea0213 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -65,11 +65,18 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i ret = taosLSeekFile(pIdxTFile, offset, SEEK_SET); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); + wError("failed to seek idx file, ver %ld, pos: %ld, since %s", ver, offset, terrstr()); return -1; } SWalIdxEntry entry; - if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + if ((ret = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry))) != sizeof(SWalIdxEntry)) { + if (ret < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("failed to read idx file, since %s", terrstr()); + } else { + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + wError("read idx file incompletely, read bytes %d, bytes should be %lu", ret, sizeof(SWalIdxEntry)); + } return -1; } @@ -77,6 +84,7 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i ret = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); + wError("failed to seek log file, ver %ld, pos: %ld, since %s", ver, entry.offset, terrstr()); return -1; } return ret; @@ -92,6 +100,8 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_READ); if (pLogTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TSDB_CODE_WAL_INVALID_VER; + wError("cannot open file %s, since %s", fnameStr, terrstr()); return -1; } @@ -99,6 +109,7 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ); if (pIdxTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); + wError("cannot open file %s, since %s", fnameStr, terrstr()); return -1; } @@ -113,6 +124,7 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { return 0; } if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { + wError("invalid version: % " PRId64 ", first ver %ld, last ver %ld", ver, pWal->vers.firstVer, pWal->vers.lastVer); terrno = TSDB_CODE_WAL_INVALID_VER; return -1; } @@ -125,12 +137,13 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); ASSERT(pRet != NULL); if (pRead->curFileFirstVer != pRet->firstVer) { + // error code set inner if (walReadChangeFile(pRead, pRet->firstVer) < 0) { return -1; } } - // code set inner + // error code set inner if (walReadSeekFilePos(pRead, pRet->firstVer, ver) < 0) { return -1; } @@ -248,8 +261,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { // TODO: check wal life if (pRead->curVersion != ver) { if (walReadSeekVer(pRead, ver) < 0) { - terrno = TSDB_CODE_WAL_INVALID_VER; - wError("unexpected wal log version: % " PRId64 ", since seek error", ver); + wError("unexpected wal log version: % " PRId64 ", since %s", ver, terrstr()); return -1; } } From f19272909c5af46e16675cc719bbd8a85a775f4e Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 12 May 2022 17:17:02 +0800 Subject: [PATCH 77/94] fix: some problems of parser --- include/libs/nodes/querynodes.h | 2 +- include/util/taoserror.h | 1 + source/libs/parser/inc/parUtil.h | 1 + source/libs/parser/src/parAstCreater.c | 2 +- source/libs/parser/src/parInsert.c | 29 ++++++++------- source/libs/parser/src/parInsertData.c | 4 +-- source/libs/parser/src/parTranslater.c | 35 +++++++++++++------ source/libs/parser/src/parUtil.c | 21 ++++++++++- source/libs/planner/src/planOptimizer.c | 20 ++++++++--- source/libs/planner/src/planUtil.c | 2 +- source/libs/planner/test/planOptimizeTest.cpp | 2 ++ 11 files changed, 87 insertions(+), 32 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index c0e2ec91e0..d07f29c487 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -232,9 +232,9 @@ typedef struct SSelectStmt { char stmtName[TSDB_TABLE_NAME_LEN]; uint8_t precision; bool isEmptyResult; + bool isTimeOrderQuery; bool hasAggFuncs; bool hasRepeatScanFuncs; - bool isTimeOrderQuery; } SSelectStmt; typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 52b2f0c670..51dc14111c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -634,6 +634,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_TAGS_NUM TAOS_DEF_ERROR_CODE(0, 0x2643) #define TSDB_CODE_PAR_PERMISSION_DENIED TAOS_DEF_ERROR_CODE(0, 0x2644) #define TSDB_CODE_PAR_INVALID_STREAM_QUERY TAOS_DEF_ERROR_CODE(0, 0x2645) +#define TSDB_CODE_PAR_INVALID_INTERNAL_PK TAOS_DEF_ERROR_CODE(0, 0x2646) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index c146d42e05..f82d29d27e 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -46,6 +46,7 @@ SSchema* getTableTagSchema(const STableMeta* pTableMeta); int32_t getNumOfColumns(const STableMeta* pTableMeta); int32_t getNumOfTags(const STableMeta* pTableMeta); STableComInfo getTableInfo(const STableMeta* pTableMeta); +STableMeta* tableMetaDup(const STableMeta* pTableMeta); int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* errMsg, int16_t startColId); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index b7a14a81c6..74a023f0cd 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -78,7 +78,7 @@ static bool checkUserName(SAstCreateContext* pCxt, SToken* pUserName) { static bool checkPassword(SAstCreateContext* pCxt, const SToken* pPasswordToken, char* pPassword) { if (NULL == pPasswordToken) { pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; - } else if (pPasswordToken->n >= (TSDB_USET_PASSWORD_LEN - 2)) { + } else if (pPasswordToken->n >= (TSDB_USET_PASSWORD_LEN + 2)) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); } else { strncpy(pPassword, pPasswordToken->z, pPasswordToken->n); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 3d069257c9..dfc56eff39 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -237,7 +237,7 @@ static int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, con return code; } -static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SName* name, char *dbFname, bool isStb) { +static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SName* name, char* dbFname, bool isStb) { SParseContext* pBasicCtx = pCxt->pComCxt; bool pass = false; @@ -252,6 +252,7 @@ static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SName* name, char *db } else { CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, name, &pCxt->pTableMeta)); + ASSERT(pCxt->pTableMeta->tableInfo.rowSize > 0); SVgroupInfo vg; CHECK_CODE( catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, name, &vg)); @@ -260,9 +261,13 @@ static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SName* name, char *db return TSDB_CODE_SUCCESS; } -static int32_t getTableMeta(SInsertParseContext* pCxt, SName* name, char *dbFname) { return getTableMetaImpl(pCxt, name, dbFname, false); } +static int32_t getTableMeta(SInsertParseContext* pCxt, SName* name, char* dbFname) { + return getTableMetaImpl(pCxt, name, dbFname, false); +} -static int32_t getSTableMeta(SInsertParseContext* pCxt, SName* name, char *dbFname) { return getTableMetaImpl(pCxt, name, dbFname, true); } +static int32_t getSTableMeta(SInsertParseContext* pCxt, SName* name, char* dbFname) { + return getTableMetaImpl(pCxt, name, dbFname, true); +} static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchema) { while (start < end) { @@ -863,7 +868,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SName* name, char* tb createSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); char stbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(&sname, stbFName); - + CHECK_CODE(getSTableMeta(pCxt, &sname, stbFName)); if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) { return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); @@ -1063,8 +1068,8 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) { // [...]; static int32_t parseInsertBody(SInsertParseContext* pCxt) { int32_t tbNum = 0; - char tbFName[TSDB_TABLE_FNAME_LEN]; - bool autoCreateTbl = false; + char tbFName[TSDB_TABLE_FNAME_LEN]; + bool autoCreateTbl = false; // for each table while (1) { @@ -1158,7 +1163,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } memcpy(tags, &pCxt->tags, sizeof(pCxt->tags)); - (*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pTableMeta, tags, tbFName, autoCreateTbl, pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj); + (*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pTableMeta, tags, tbFName, autoCreateTbl, + pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj); memset(&pCxt->tags, 0, sizeof(pCxt->tags)); pCxt->pVgroupsHashObj = NULL; @@ -1479,7 +1485,6 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu taosMemoryFree(pSTSchema); } #endif - } if (rowEnd) { @@ -1677,10 +1682,10 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsFormat, SArray* cols buildCreateTbReq(&smlHandle->createTblReq, tableName, row, pTableMeta->suid); STableDataBlocks* pDataBlock = NULL; - ret = getDataBlockFromList(smlHandle->pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid), TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, pTableMeta, - &pDataBlock, NULL, &smlHandle->createTblReq); - if(ret != TSDB_CODE_SUCCESS){ + ret = getDataBlockFromList(smlHandle->pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid), + TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, + pTableMeta, &pDataBlock, NULL, &smlHandle->createTblReq); + if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "create data block error"); return ret; } diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 8deaad6091..677dbca0e9 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -137,7 +137,7 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star } memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); - dataBuf->pTableMeta = pTableMeta; + dataBuf->pTableMeta = tableMetaDup(pTableMeta); SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo; SSchema* pSchema = getTableColumnSchema(dataBuf->pTableMeta); @@ -465,7 +465,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p taosMemoryFreeClear(blkKeyInfo.pKeyTuple); return ret; } - + ASSERT(pOneTableBlock->pTableMeta->tableInfo.rowSize > 0); // the maximum expanded size in byte when a row-wise data is converted to SDataRow format int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3fed68188d..c86c1ac2e9 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -368,11 +368,15 @@ static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* p return TSDB_CODE_SUCCESS; } +static bool isInternalPrimaryKey(const SColumnNode* pCol) { + return PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId && 0 == strcmp(pCol->colName, PK_TS_COL_INTERNAL_NAME); +} + static bool findAndSetColumn(SColumnNode* pCol, const STableNode* pTable) { bool found = false; if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; - if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId && 0 == strcmp(pCol->colName, PK_TS_COL_INTERNAL_NAME)) { + if (isInternalPrimaryKey(pCol)) { setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema, false, pCol); return true; } @@ -389,7 +393,9 @@ static bool findAndSetColumn(SColumnNode* pCol, const STableNode* pTable) { SNode* pNode; FOREACH(pNode, pProjectList) { SExprNode* pExpr = (SExprNode*)pNode; - if (0 == strcmp(pCol->colName, pExpr->aliasName)) { + if (0 == strcmp(pCol->colName, pExpr->aliasName) || + ((QUERY_NODE_COLUMN == nodeType(pExpr) && PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pExpr)->colId) && + isInternalPrimaryKey(pCol))) { setColumnInfoByExpr(pTable, pExpr, pCol); found = true; break; @@ -433,7 +439,11 @@ static EDealRes translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNod } } if (!found) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, pCol->colName); + if (isInternalPrimaryKey(pCol)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_INTERNAL_PK); + } else { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, pCol->colName); + } } return DEAL_RES_CONTINUE; } @@ -3655,6 +3665,9 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* destroyCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; } + if (pStmt->ignoreExists) { + req.flags |= TD_CREATE_IF_NOT_EXISTS; + } SNode* pCol; col_id_t index = 0; FOREACH(pCol, pStmt->pCols) { @@ -3785,24 +3798,27 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { return code; } -static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, const char* pDbName, - const char* pTableName, SKVRow row, uint64_t suid, SVgroupInfo* pVgInfo) { +static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, SCreateSubTableClause* pStmt, SKVRow row, + uint64_t suid, SVgroupInfo* pVgInfo) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; - strcpy(name.dbname, pDbName); + strcpy(name.dbname, pStmt->dbName); tNameGetFullDbName(&name, dbFName); struct SVCreateTbReq req = {0}; req.type = TD_CHILD_TABLE; - req.name = strdup(pTableName); + req.name = strdup(pStmt->tableName); req.ctb.suid = suid; req.ctb.pTag = row; + if (pStmt->ignoreExists) { + req.flags |= TD_CREATE_IF_NOT_EXISTS; + } SVgroupCreateTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId)); if (pTableBatch == NULL) { SVgroupCreateTableBatch tBatch = {0}; tBatch.info = *pVgInfo; - strcpy(tBatch.dbName, pDbName); + strcpy(tBatch.dbName, pStmt->dbName); tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq)); taosArrayPush(tBatch.req.pArray, &req); @@ -3964,8 +3980,7 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla code = getTableHashVgroup(pCxt, pStmt->dbName, pStmt->tableName, &info); } if (TSDB_CODE_SUCCESS == code) { - addCreateTbReqIntoVgroup(pCxt->pParseCxt->acctId, pVgroupHashmap, pStmt->dbName, pStmt->tableName, row, - pSuperTableMeta->uid, &info); + addCreateTbReqIntoVgroup(pCxt->pParseCxt->acctId, pVgroupHashmap, pStmt, row, pSuperTableMeta->uid, &info); } taosMemoryFreeClear(pSuperTableMeta); diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 715b9b97e6..43aea8de7c 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -146,6 +146,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Invalid binary/nchar column length"; case TSDB_CODE_PAR_INVALID_TAGS_NUM: return "Invalid number of tag columns"; + case TSDB_CODE_PAR_INVALID_INTERNAL_PK: + return "Invalid _c0 or _rowts expression"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: @@ -191,7 +193,7 @@ int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } -SSchema *getTableColumnSchema(const STableMeta *pTableMeta) { +SSchema* getTableColumnSchema(const STableMeta* pTableMeta) { assert(pTableMeta != NULL); return (SSchema*)pTableMeta->schema; } @@ -226,6 +228,23 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta) { return pTableMeta->tableInfo; } +static uint32_t getTableMetaSize(const STableMeta* pTableMeta) { + int32_t totalCols = 0; + if (pTableMeta->tableInfo.numOfColumns >= 0) { + totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; + } + + return sizeof(STableMeta) + totalCols * sizeof(SSchema); +} + +STableMeta* tableMetaDup(const STableMeta* pTableMeta) { + size_t size = getTableMetaSize(pTableMeta); + + STableMeta* p = taosMemoryMalloc(size); + memcpy(p, pTableMeta, size); + return p; +} + int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) { if (len <= 0 || dlen <= 0) return 0; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 30f6f03a6c..5ed7d9c1b5 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -151,20 +151,32 @@ static bool needOptimizeDynamicScan(const SFunctionNode* pFunc) { static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, SNodeList** pDsoFuncs) { SNodeList* pAllFuncs = osdGetAllFuncs(pScan->node.pParent); + SNodeList* pTmpSdrFuncs = NULL; + SNodeList* pTmpDsoFuncs = NULL; SNode* pFunc = NULL; + bool otherFunc = false; FOREACH(pFunc, pAllFuncs) { int32_t code = TSDB_CODE_SUCCESS; if (needOptimizeDataRequire((SFunctionNode*)pFunc)) { - code = nodesListMakeStrictAppend(pSdrFuncs, nodesCloneNode(pFunc)); + code = nodesListMakeStrictAppend(&pTmpSdrFuncs, nodesCloneNode(pFunc)); } else if (needOptimizeDynamicScan((SFunctionNode*)pFunc)) { - code = nodesListMakeStrictAppend(pDsoFuncs, nodesCloneNode(pFunc)); + code = nodesListMakeStrictAppend(&pTmpDsoFuncs, nodesCloneNode(pFunc)); + } else { + otherFunc = true; } if (TSDB_CODE_SUCCESS != code) { - nodesDestroyList(*pSdrFuncs); - nodesDestroyList(*pDsoFuncs); + nodesDestroyList(pTmpSdrFuncs); + nodesDestroyList(pTmpDsoFuncs); return code; } } + if (otherFunc) { + nodesDestroyList(pTmpSdrFuncs); + nodesDestroyList(pTmpDsoFuncs); + } else { + *pSdrFuncs = pTmpSdrFuncs; + *pDsoFuncs = pTmpDsoFuncs; + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index 36625b28fb..3c83d9f53a 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -18,7 +18,7 @@ static char* getUsageErrFormat(int32_t errCode) { switch (errCode) { case TSDB_CODE_PLAN_EXPECTED_TS_EQUAL: - return "l.ts = r.ts is expected in join expression"; + return "left.ts = right.ts is expected in join expression"; case TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN: return "not support cross join"; default: diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index 6c7b1d0a0e..77f9b5846c 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -28,6 +28,8 @@ TEST_F(PlanOptimizeTest, optimizeScanData) { run("SELECT COUNT(c1) FROM t1"); run("SELECT COUNT(CAST(c1 AS BIGINT)) FROM t1"); + + run("SELECT PERCENTILE(c1, 40), COUNT(*) FROM t1"); } TEST_F(PlanOptimizeTest, orderByPrimaryKey) { From 2d89555b724646ef4a944a6bc1a5b3e65fa13a84 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 May 2022 17:38:16 +0800 Subject: [PATCH 78/94] fix(rpc): avoid fd leak --- source/libs/qcom/src/queryUtil.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 3e3e393f5f..4b4c079649 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -153,11 +153,6 @@ int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTra .handle = pInfo->msgInfo.handle, .persistHandle = persistHandle, .code = 0}; - if (pInfo->msgType == TDMT_VND_QUERY || pInfo->msgType == TDMT_VND_FETCH || - pInfo->msgType == TDMT_VND_QUERY_CONTINUE) { - rpcMsg.persistHandle = 1; - } - assert(pInfo->fp != NULL); rpcSendRequestWithCtx(pTransporter, epSet, &rpcMsg, pTransporterId, rpcCtx); @@ -168,7 +163,7 @@ int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransp return asyncSendMsgToServerExt(pTransporter, epSet, pTransporterId, pInfo, false, NULL); } -char *jobTaskStatusStr(int32_t status) { +char* jobTaskStatusStr(int32_t status) { switch (status) { case JOB_TASK_STATUS_NULL: return "NULL"; @@ -197,13 +192,10 @@ char *jobTaskStatusStr(int32_t status) { SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name) { SSchema s = {0}; - s.type = type; + s.type = type; s.bytes = bytes; s.colId = colId; tstrncpy(s.name, name, tListLen(s.name)); return s; } - - - From ae8b3cc11d7980076b42626159caebf1e47f6655 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 12 May 2022 17:45:17 +0800 Subject: [PATCH 79/94] enh(tsdb): return oom err for submit msg --- source/dnode/vnode/src/vnd/vnodeSvr.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index fc2b6fe676..d2650725f0 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -562,6 +562,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in int32_t nRows; int32_t tsize, ret; SEncoder encoder = {0}; + terrno = TSDB_CODE_SUCCESS; pRsp->code = 0; @@ -576,6 +577,11 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in } submitRsp.pArray = taosArrayInit(pSubmitReq->numOfBlocks, sizeof(SSubmitBlkRsp)); + if (!submitRsp.pArray) { + pRsp->code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + for (int i = 0;;) { tGetSubmitMsgNext(&msgIter, &pBlock); if (pBlock == NULL) break; @@ -640,7 +646,12 @@ _exit: taosArrayDestroy(submitRsp.pArray); - tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + // TODO: the partial success scenario and the error case + // TODO: refactor + if ((terrno == TSDB_CODE_SUCCESS || terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) && + (pRsp->code == TSDB_CODE_SUCCESS)) { + tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + } return 0; } From 7771023d7b70e76b8006015e47a208da52996681 Mon Sep 17 00:00:00 2001 From: dapan Date: Thu, 12 May 2022 18:02:57 +0800 Subject: [PATCH 80/94] string to timestamp --- include/common/ttypes.h | 1 + source/libs/scalar/inc/sclInt.h | 1 + source/libs/scalar/src/filter.c | 16 ++-------------- source/libs/scalar/src/scalar.c | 27 +++++++++++++++++++++++++-- 4 files changed, 29 insertions(+), 16 deletions(-) diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 377b443843..cab429d136 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -50,6 +50,7 @@ typedef struct { #define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len)) #define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON)) +#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR)) #define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 659d7dcf7e..1e7d1a4cbf 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -49,6 +49,7 @@ typedef struct SScalarCtx { int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out); SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows); +void sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode); #define GET_PARAM_TYPE(_c) ((_c)->columnData->info.type) #define GET_PARAM_BYTES(_c) ((_c)->columnData->info.bytes) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index eaab8e1f53..768cf51d25 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3505,18 +3505,6 @@ int32_t fltAddValueNodeToConverList(SFltTreeStat *stat, SValueNode* pNode) { return TSDB_CODE_SUCCESS; } -void fltConvertToTsValueNode(SFltTreeStat *stat, SValueNode* valueNode) { - char *timeStr = valueNode->datum.p; - if (convertStringToTimestamp(valueNode->node.resType.type, valueNode->datum.p, stat->precision, &valueNode->datum.i) != - TSDB_CODE_SUCCESS) { - valueNode->datum.i = 0; - } - taosMemoryFree(timeStr); - - valueNode->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; - valueNode->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; -} - EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { SFltTreeStat *stat = (SFltTreeStat *)pContext; @@ -3565,7 +3553,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } - fltConvertToTsValueNode(stat, valueNode); + sclConvertToTsValueNode(stat->precision, valueNode); return DEAL_RES_CONTINUE; } @@ -3694,7 +3682,7 @@ int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { for (int32_t i = 0; i < nodeNum; ++i) { SValueNode *valueNode = *(SValueNode **)taosArrayGet(pStat->nodeList, i); - fltConvertToTsValueNode(pStat, valueNode); + sclConvertToTsValueNode(pStat->precision, valueNode); } _return: diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 5231890821..92d3a5a54c 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -19,6 +19,19 @@ int32_t scalarGetOperatorParamNum(EOperatorType type) { return 2; } +void sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode) { + char *timeStr = valueNode->datum.p; + if (convertStringToTimestamp(valueNode->node.resType.type, valueNode->datum.p, precision, &valueNode->datum.i) != + TSDB_CODE_SUCCESS) { + valueNode->datum.i = 0; + } + taosMemoryFree(timeStr); + + valueNode->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; + valueNode->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; +} + + SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) { SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData)); if (pColumnData == NULL) { @@ -534,7 +547,7 @@ EDealRes sclRewriteBasedOnOptr(SNode** pNode, SScalarCtx *ctx, EOperatorType opT } -EDealRes sclRewriteOperatorForNullValue(SNode** pNode, SScalarCtx *ctx) { +EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { SOperatorNode *node = (SOperatorNode *)*pNode; if (node->pLeft && (QUERY_NODE_VALUE == nodeType(node->pLeft))) { @@ -542,6 +555,11 @@ EDealRes sclRewriteOperatorForNullValue(SNode** pNode, SScalarCtx *ctx) { if (SCL_IS_NULL_VALUE_NODE(valueNode) && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) { return sclRewriteBasedOnOptr(pNode, ctx, node->opType); } + + if (IS_STR_DATA_TYPE(valueNode->node.resType.type) && node->pRight && nodesIsExprNode(node->pRight) + && ((SExprNode*)node->pRight)->resType.type == TSDB_DATA_TYPE_TIMESTAMP) { + sclConvertToTsValueNode(((SExprNode*)node->pRight)->resType.precision, valueNode); + } } if (node->pRight && (QUERY_NODE_VALUE == nodeType(node->pRight))) { @@ -549,6 +567,11 @@ EDealRes sclRewriteOperatorForNullValue(SNode** pNode, SScalarCtx *ctx) { if (SCL_IS_NULL_VALUE_NODE(valueNode) && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) { return sclRewriteBasedOnOptr(pNode, ctx, node->opType); } + + if (IS_STR_DATA_TYPE(valueNode->node.resType.type) && node->pLeft && nodesIsExprNode(node->pLeft) + && ((SExprNode*)node->pLeft)->resType.type == TSDB_DATA_TYPE_TIMESTAMP) { + sclConvertToTsValueNode(((SExprNode*)node->pLeft)->resType.precision, valueNode); + } } if (node->pRight && (QUERY_NODE_NODE_LIST == nodeType(node->pRight))) { @@ -671,7 +694,7 @@ EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { SOperatorNode *node = (SOperatorNode *)*pNode; if ((!SCL_IS_CONST_NODE(node->pLeft)) || (!SCL_IS_CONST_NODE(node->pRight))) { - return sclRewriteOperatorForNullValue(pNode, ctx); + return sclRewriteNonConstOperator(pNode, ctx); } SScalarParam output = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; From d66bb65171a96a80e413684a4f0fd941eca8d6d0 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 12 May 2022 18:32:32 +0800 Subject: [PATCH 81/94] feat(tsdb): unit of days and keep support minute --- include/common/tglobal.h | 4 ++-- source/common/src/tglobal.c | 10 +++++----- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 8 ++++---- source/dnode/vnode/src/inc/tsdb.h | 8 ++++---- source/dnode/vnode/src/tsdb/tsdbCommit.c | 9 ++++----- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +++--- source/dnode/vnode/src/tsdb/tsdbSma.c | 2 +- source/dnode/vnode/src/tsdb/tsdbWrite.c | 4 ++-- 8 files changed, 25 insertions(+), 26 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 97e6491b98..84aae46347 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -43,7 +43,7 @@ extern int32_t tsMaxNumOfDistinctResults; extern int32_t tsCompatibleModel; extern bool tsEnableSlaveQuery; extern bool tsPrintAuth; -extern int64_t tsTickPerDay[3]; +extern int64_t tsTickPerMin[3]; // multi-process extern bool tsMultiProcess; @@ -122,7 +122,7 @@ extern int32_t tsDiskCfgNum; extern SDiskCfg tsDiskCfg[]; // udf -extern bool tsStartUdfd; +extern bool tsStartUdfd; // internal extern int32_t tsTransPullupInterval; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 42c8e18f3e..0999cb4d2c 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -153,11 +153,11 @@ bool tsStreamSchedV = true; /* * minimum scale for whole system, millisecond by default - * for TSDB_TIME_PRECISION_MILLI: 86400000L - * TSDB_TIME_PRECISION_MICRO: 86400000000L - * TSDB_TIME_PRECISION_NANO: 86400000000000L + * for TSDB_TIME_PRECISION_MILLI: 60000L + * TSDB_TIME_PRECISION_MICRO: 60000000L + * TSDB_TIME_PRECISION_NANO: 60000000000L */ -int64_t tsTickPerDay[] = {86400000L, 86400000000L, 86400000000000L}; +int64_t tsTickPerMin[] = {60000L, 60000000L, 60000000000L}; // lossy compress 6 char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty @@ -170,7 +170,7 @@ uint32_t tsCurRange = 100; // range char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR // udf -bool tsStartUdfd = true; +bool tsStartUdfd = true; // internal int32_t tsTransPullupInterval = 6; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 6a0f4984b1..ae0f1b0d55 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -145,10 +145,10 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->szBuf = pCreate->buffer * 1024 * 1024; pCfg->isWeak = true; pCfg->tsdbCfg.precision = pCreate->precision; - pCfg->tsdbCfg.days = 10; - pCfg->tsdbCfg.keep0 = 3650; - pCfg->tsdbCfg.keep1 = 3650; - pCfg->tsdbCfg.keep2 = 3650; + pCfg->tsdbCfg.days = pCreate->daysPerFile; + pCfg->tsdbCfg.keep0 = pCreate->daysToKeep0; + pCfg->tsdbCfg.keep1 = pCreate->daysToKeep1; + pCfg->tsdbCfg.keep2 = pCreate->daysToKeep2; pCfg->tsdbCfg.minRows = pCreate->minRows; pCfg->tsdbCfg.maxRows = pCreate->maxRows; for (size_t i = 0; i < taosArrayGetSize(pCreate->pRetensions); ++i) { diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index b8cbb2d997..c9305f6af9 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -518,9 +518,9 @@ void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn); static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { if (key < 0) { - return (int)((key + 1) / tsTickPerDay[precision] / days - 1); + return (int)((key + 1) / tsTickPerMin[precision] / days - 1); } else { - return (int)((key / tsTickPerDay[precision] / days)); + return (int)((key / tsTickPerMin[precision] / days)); } } @@ -770,8 +770,8 @@ static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet *pSrc, SDFileSet *pDest) { } static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY *minKey, TSKEY *maxKey) { - *minKey = fid * days * tsTickPerDay[precision]; - *maxKey = *minKey + days * tsTickPerDay[precision] - 1; + *minKey = fid * days * tsTickPerMin[precision]; + *maxKey = *minKey + days * tsTickPerMin[precision] - 1; } static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet *pSet) { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index c1813c787a..d180799e58 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -216,9 +216,9 @@ void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) { TSKEY minKey, midKey, maxKey, now; now = taosGetTimestamp(pCfg->precision); - minKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision]; - midKey = now - pCfg->keep1 * tsTickPerDay[pCfg->precision]; - maxKey = now - pCfg->keep0 * tsTickPerDay[pCfg->precision]; + minKey = now - pCfg->keep2 * tsTickPerMin[pCfg->precision]; + midKey = now - pCfg->keep1 * tsTickPerMin[pCfg->precision]; + maxKey = now - pCfg->keep0 * tsTickPerMin[pCfg->precision]; pRtn->minKey = minKey; pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->days, pCfg->precision)); @@ -398,7 +398,7 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { ++mIter; } else if (pIter && !pIter->pTable) { // When table already dropped during commit, pIter is not NULL but pIter->pTable is NULL. - ++mIter; // skip the table and do nothing + ++mIter; // skip the table and do nothing } else if (pIdx) { if (tsdbMoveBlkIdx(pCommith, pIdx) < 0) { tsdbCloseCommitFile(pCommith, true); @@ -948,7 +948,6 @@ static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx) { static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable) { STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); - pCommith->pTable = pTable; if (tdInitDataCols(pCommith->pDataCols, pSchema) < 0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 2511e3a570..b293f1399d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -323,7 +323,7 @@ static int64_t getEarliestValidTimestamp(STsdb* pTsdb) { STsdbKeepCfg* pCfg = REPO_KEEP_CFG(pTsdb); int64_t now = taosGetTimestamp(pCfg->precision); - return now - (tsTickPerDay[pCfg->precision] * pCfg->keep2) + 1; // needs to add one tick + return now - (tsTickPerMin[pCfg->precision] * pCfg->keep2) + 1; // needs to add one tick } static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableDataCond* pCond) { @@ -1047,10 +1047,10 @@ static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precisio } if (key < 0) { - key -= (daysPerFile * tsTickPerDay[precision]); + key -= (daysPerFile * tsTickPerMin[precision]); } - int64_t fid = (int64_t)(key / (daysPerFile * tsTickPerDay[precision])); // set the starting fileId + int64_t fid = (int64_t)(key / (daysPerFile * tsTickPerMin[precision])); // set the starting fileId if (fid < 0L && llabs(fid) > INT32_MAX) { // data value overflow for INT32 fid = INT32_MIN; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 61515a3be1..e878668654 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -1017,7 +1017,7 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe int32_t daysPerFile = pCfg->days; if (storageLevel == SMA_STORAGE_LEVEL_TSDB) { - int32_t days = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerDay[pCfg->precision]); + int32_t days = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]); daysPerFile = days > SMA_STORAGE_TSDB_DAYS ? days : SMA_STORAGE_TSDB_DAYS; } diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 3107c6f5c7..341ab94ca4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -63,8 +63,8 @@ static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { STSRow *row = NULL; STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pTsdb); TSKEY now = taosGetTimestamp(pCfg->precision); - TSKEY minKey = now - tsTickPerDay[pCfg->precision] * pCfg->keep2; - TSKEY maxKey = now + tsTickPerDay[pCfg->precision] * pCfg->days; + TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2; + TSKEY maxKey = now + tsTickPerMin[pCfg->precision] * pCfg->days; terrno = TSDB_CODE_SUCCESS; // pMsg->length = htonl(pMsg->length); From 910436a5647723b76852037ed29ec71299b51142 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 12 May 2022 18:46:20 +0800 Subject: [PATCH 82/94] feat(tsdb): comp use cfg from params --- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index ae0f1b0d55..914acce2ea 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -144,6 +144,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->szCache = pCreate->pages; pCfg->szBuf = pCreate->buffer * 1024 * 1024; pCfg->isWeak = true; + pCfg->tsdbCfg.compression = pCreate->compression; pCfg->tsdbCfg.precision = pCreate->precision; pCfg->tsdbCfg.days = pCreate->daysPerFile; pCfg->tsdbCfg.keep0 = pCreate->daysToKeep0; From d6f03eb7da95209406a1bb641d6b42a24d980330 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 12 May 2022 18:59:02 +0800 Subject: [PATCH 83/94] fix: some problems of parser --- source/libs/parser/src/parInsert.c | 24 +++++++++++++++---- source/libs/scheduler/src/scheduler.c | 33 +++++++++++++++++---------- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index dfc56eff39..9c788202d3 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -53,6 +53,7 @@ typedef struct SInsertParseContext { SHashObj* pTableBlockHashObj; // global SHashObj* pSubTableHashObj; // global SArray* pVgDataBlocks; // global + SHashObj* pTableNameHashObj; // global int32_t totalNum; SVnodeModifOpStmt* pOutput; SStmtCallback* pStmtCb; @@ -1111,6 +1112,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { createSName(&name, &tbnameToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); tNameExtractFullName(&name, tbFName); + CHECK_CODE(taosHashPut(pCxt->pTableNameHashObj, tbFName, strlen(tbFName), &name, sizeof(SName))); + // USING cluase if (TK_USING == sToken.type) { CHECK_CODE(parseUsingClause(pCxt, &name, tbFName)); @@ -1193,7 +1196,8 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { .pSql = (char*)pContext->pSql, .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, .pTableMeta = NULL, - .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, false), + .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK), + .pTableNameHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK), .totalNum = 0, .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT), .pStmtCb = pContext->pStmtCb}; @@ -1202,12 +1206,13 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { (*pContext->pStmtCb->getExecInfoFn)(pContext->pStmtCb->pStmt, &context.pVgroupsHashObj, &context.pTableBlockHashObj); } else { - context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - context.pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + context.pTableBlockHashObj = + taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || NULL == context.pSubTableHashObj || - NULL == context.pOutput) { + NULL == context.pTableNameHashObj || NULL == context.pOutput) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1220,6 +1225,10 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { if (NULL == *pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } + (*pQuery)->pTableList = taosArrayInit(taosHashGetSize(context.pTableNameHashObj), sizeof(SName)); + if (NULL == (*pQuery)->pTableList) { + return TSDB_CODE_OUT_OF_MEMORY; + } (*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE; (*pQuery)->haveResultSet = false; (*pQuery)->msgType = TDMT_VND_SUBMIT; @@ -1232,6 +1241,13 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { if (TSDB_CODE_SUCCESS == code) { code = parseInsertBody(&context); } + if (TSDB_CODE_SUCCESS == code) { + SName* pTable = taosHashIterate(context.pTableNameHashObj, NULL); + while (NULL != pTable) { + taosArrayPush((*pQuery)->pTableList, pTable); + pTable = taosHashIterate(context.pTableNameHashObj, pTable); + } + } destroyInsertParseContext(&context); return code; } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 1ce074c49f..2710e54f95 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1092,11 +1092,10 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch if (TSDB_CODE_SUCCESS == code && batchRsp.nRsps > 0) { for (int32_t i = 0; i < batchRsp.nRsps; ++i) { SVCreateTbRsp *rsp = batchRsp.pRsps + i; - if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) { - tDecoderClear(&coder); - SCH_ERR_JRET(rsp->code); - } else if (TSDB_CODE_SUCCESS != rsp->code) { + if (TSDB_CODE_SUCCESS != rsp->code) { code = rsp->code; + tDecoderClear(&coder); + SCH_ERR_JRET(code); } } } @@ -1117,11 +1116,10 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch if (TSDB_CODE_SUCCESS == code && batchRsp.nRsps > 0) { for (int32_t i = 0; i < batchRsp.nRsps; ++i) { SVDropTbRsp *rsp = batchRsp.pRsps + i; - if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) { - tDecoderClear(&coder); - SCH_ERR_JRET(rsp->code); - } else if (TSDB_CODE_SUCCESS != rsp->code) { + if (TSDB_CODE_SUCCESS != rsp->code) { code = rsp->code; + tDecoderClear(&coder); + SCH_ERR_JRET(code); } } } @@ -1137,7 +1135,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCH_ERR_JRET(rspCode); if (msg) { - SDecoder coder = {0}; + SDecoder coder = {0}; SSubmitRsp *rsp = taosMemoryMalloc(sizeof(*rsp)); tDecoderInit(&coder, msg, msgSize); code = tDecodeSSubmitRsp(&coder, rsp); @@ -1146,10 +1144,21 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch tFreeSSubmitRsp(rsp); SCH_ERR_JRET(code); } - + + if (rsp->nBlocks > 0) { + for (int32_t i = 0; i < rsp->nBlocks; ++i) { + SSubmitBlkRsp *blk = rsp->pBlocks + i; + if (TSDB_CODE_SUCCESS != blk->code) { + code = blk->code; + tFreeSSubmitRsp(rsp); + SCH_ERR_JRET(code); + } + } + } + atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows); SCH_TASK_DLOG("submit succeed, affectedRows:%d", rsp->affectedRows); - + if (pJob->attr.needRes) { SCH_LOCK(SCH_WRITE, &pJob->resLock); if (pJob->resData) { @@ -1160,7 +1169,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch memcpy(sum->pBlocks + sum->nBlocks - rsp->nBlocks, rsp->pBlocks, rsp->nBlocks * sizeof(*sum->pBlocks)); taosMemoryFree(rsp->pBlocks); taosMemoryFree(rsp); - } else { + } else { pJob->resData = rsp; } SCH_UNLOCK(SCH_WRITE, &pJob->resLock); From c55d20b19159f1c777d1234a0cee716d656d386a Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 12 May 2022 19:09:53 +0800 Subject: [PATCH 84/94] test: error script for debug --- tests/system-test/99-TDcase/TD-15517.py | 372 ++++++++++++++++++++++++ 1 file changed, 372 insertions(+) create mode 100644 tests/system-test/99-TDcase/TD-15517.py diff --git a/tests/system-test/99-TDcase/TD-15517.py b/tests/system-test/99-TDcase/TD-15517.py new file mode 100644 index 0000000000..e7837ba009 --- /dev/null +++ b/tests/system-test/99-TDcase/TD-15517.py @@ -0,0 +1,372 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + hostname = socket.gethostname() + #rpcDebugFlagVal = '143' + #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #print ("===================: ", updatecfgDict) + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def newcur(self,cfg,host,port): + user = "root" + password = "taosdata" + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): + tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("use %s" %dbName) + tsql.execute("create table %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) + pre_create = "create table" + sql = pre_create + #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + for i in range(ctbNum): + sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1) + if (i > 0) and (i%100 == 0): + tsql.execute(sql) + sql = pre_create + if sql != pre_create: + tsql.execute(sql) + + tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum)) + return + + def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + for i in range(ctbNum): + sql += " %s_%d values "%(stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + if j < rowsPerTbl - 1: + sql = "insert into %s_%d values " %(stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareEnv(self, **parameterDict): + print ("input parameters:") + print (parameterDict) + # create new connector for my thread + tsql=self.newcur(parameterDict['cfg'], 'localhost', 6030) + self.create_tables(tsql,\ + parameterDict["dbName"],\ + parameterDict["vgroups"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"]) + + self.insert_data(tsql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"],\ + parameterDict["startTs"]) + return + + def run(self): + tdSql.prepare() + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + tdLog.printNoPrefix("======== test scenario 1: ") + tdLog.info("step 1: create database, stb, ctb and insert data") + # create and start thread + parameterDict = {'cfg': '', \ + 'dbName': 'db', \ + 'vgroups': 1, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 10, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + time.sleep(2) + + # wait stb ready + while 1: + tdSql.query("show %s.stables"%parameterDict['dbName']) + if tdSql.getRows() == 1: + break + else: + time.sleep(1) + + tdLog.info("create topics from super table") + topicFromStb = 'topic_stb_column' + topicFromCtb = 'topic_ctb_column' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName'])) + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName'])) + + time.sleep(1) + tdSql.query("show topics") + #tdSql.checkRows(2) + topic1 = tdSql.getData(0 , 0) + topic2 = tdSql.getData(1 , 0) + print (topic1) + print (topic2) + + print (topicFromStb) + print (topicFromCtb) + #tdLog.info("show topics: %s, %s"%topic1, topic2) + #if topic1 != topicFromStb or topic1 != topicFromCtb: + # tdLog.exit("topic error1") + #if topic2 != topicFromStb or topic2 != topicFromCtb: + # tdLog.exit("topic error2") + + tdLog.info("create consume info table and consume result table") + cdbName = parameterDict["dbName"] + tdSql.query("create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)") + tdSql.query("create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)") + + consumerId = 0 + expectmsgcnt = (parameterDict["rowsPerTbl"] / parameterDict["batchNum"] ) * parameterDict["ctbNum"] + expectmsgcnt1 = expectmsgcnt + parameterDict["ctbNum"] + topicList = topicFromStb + ifcheckdata = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + sql = "insert into consumeinfo values " + sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectmsgcnt1, ifcheckdata) + tdSql.query(sql) + + tdLog.info("check stb if there are data") + while 1: + tdSql.query("select count(*) from %s"%parameterDict["stbName"]) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + countOfStb = tdSql.getData(0, 0) + if countOfStb != 0: + tdLog.info("count from stb: %d"%countOfStb) + break + else: + time.sleep(1) + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" + tdLog.info(shellCmd) + os.system(shellCmd) + + # wait for data ready + prepareEnvThread.join() + + tdLog.info("insert process end, and start to check consume result") + while 1: + tdSql.query("select * from consumeresult") + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == 1: + break + else: + time.sleep(5) + + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + + tdSql.checkData(0 , 1, consumerId) + tdSql.checkData(0 , 2, expectmsgcnt) + tdSql.checkData(0 , 3, expectrowcnt) + + tdSql.query("drop topic %s"%topicFromStb) + tdSql.query("drop topic %s"%topicFromCtb) + + # ============================================================================== + tdLog.printNoPrefix("======== test scenario 2: add child table with consuming ") + tdLog.info(" clean database") + # create and start thread + parameterDict = {'cfg': '', \ + 'dbName': 'db2', \ + 'vgroups': 1, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + # wait db ready + while 1: + tdSql.query("show databases") + if tdSql.getRows() == 3: + print (tdSql.getData(0,0), tdSql.getData(1,0),tdSql.getData(2,0),) + break + else: + time.sleep(1) + + tdSql.query("use %s"%parameterDict['dbName']) + # wait stb ready + while 1: + tdSql.query("show %s.stables"%parameterDict['dbName']) + if tdSql.getRows() == 1: + break + else: + time.sleep(1) + + tdLog.info("create topics from super table") + topicFromStb = 'topic_stb_column2' + topicFromCtb = 'topic_ctb_column2' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName'])) + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName'])) + + time.sleep(1) + tdSql.query("show topics") + topic1 = tdSql.getData(0 , 0) + topic2 = tdSql.getData(1 , 0) + print (topic1) + print (topic2) + + print (topicFromStb) + print (topicFromCtb) + #tdLog.info("show topics: %s, %s"%topic1, topic2) + #if topic1 != topicFromStb or topic1 != topicFromCtb: + # tdLog.exit("topic error1") + #if topic2 != topicFromStb or topic2 != topicFromCtb: + # tdLog.exit("topic error2") + + tdLog.info("create consume info table and consume result table") + cdbName = parameterDict["dbName"] + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + + consumerId = 0 + expectmsgcnt = (parameterDict["rowsPerTbl"] / parameterDict["batchNum"] ) * parameterDict["ctbNum"] + expectmsgcnt1 = expectmsgcnt + parameterDict["ctbNum"] + topicList = topicFromStb + ifcheckdata = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + sql = "insert into consumeinfo values " + sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectmsgcnt1, ifcheckdata) + tdSql.query(sql) + + tdLog.info("check stb if there are data") + while 1: + tdSql.query("select count(*) from %s"%parameterDict["stbName"]) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + countOfStb = tdSql.getData(0, 0) + if countOfStb != 0: + tdLog.info("count from stb: %d"%countOfStb) + break + else: + time.sleep(1) + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" + tdLog.info(shellCmd) + os.system(shellCmd) + + # create new child table and insert data + newCtbName = 'newctb' + rowsOfNewCtb = 1000 + tdSql.query("create table %s.%s using %s.%s tags(9999)"%(parameterDict["dbName"], newCtbName, parameterDict["dbName"], parameterDict["stbName"])) + startTs = parameterDict["startTs"] + for j in range(rowsOfNewCtb): + sql = "insert into %s.%s values (%d, %d, 'tmqrow_%d') "%(parameterDict["dbName"], newCtbName, startTs + j, j, j) + tdSql.execute(sql) + tdLog.debug("insert data into new child table ............ [OK]") + + # wait for data ready + prepareEnvThread.join() + + tdLog.info("insert process end, and start to check consume result") + while 1: + tdSql.query("select * from consumeresult") + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == 1: + break + else: + time.sleep(5) + + expectmsgcnt += rowsOfNewCtb + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + rowsOfNewCtb + + tdSql.checkData(0 , 1, consumerId) + tdSql.checkData(0 , 2, expectmsgcnt) + tdSql.checkData(0 , 3, expectrowcnt) + + + # ============================================================================== + tdLog.printNoPrefix("======== test scenario 3: ") + + #os.system('pkill tmq_sim') + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 33bd196ff04556deba9e7091178b38603ce3e39d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 12 May 2022 19:20:25 +0800 Subject: [PATCH 85/94] stmt query --- source/libs/planner/src/planner.c | 2 +- tests/script/api/batchprepare.c | 246 +++++++++++++++++------------- 2 files changed, 138 insertions(+), 110 deletions(-) diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 9ef4d80660..f65e674bf6 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -224,7 +224,7 @@ static int32_t calcConstList(SNodeList* pList) { } static bool isEmptyResultCond(SNode** pCond) { - if (QUERY_NODE_VALUE != nodeType(*pCond)) { + if (NULL == *pCond || QUERY_NODE_VALUE != nodeType(*pCond)) { return false; } if (((SValueNode*)*pCond)->datum.b) { diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index adf38015ae..ae07b3fc8e 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -20,6 +20,11 @@ typedef struct { bool enclose; } OperInfo; +typedef struct { + char* funcName; + int32_t paramNum; +} FuncInfo; + typedef enum { BP_BIND_TAG = 1, BP_BIND_COL, @@ -44,6 +49,13 @@ OperInfo operInfo[] = { int32_t operatorList[] = {0, 1, 2, 3, 4, 5, 6, 7}; int32_t varoperatorList[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; +FuncInfo funcInfo[] = { + {"count", 1}, + {"sum", 1}, + {"min", 1}, + {"sin", 1}, +}; + char *bpStbPrefix = "st"; char *bpTbPrefix = "t"; int32_t bpDefaultStbId = 1; @@ -154,7 +166,7 @@ CaseCfg gCase[] = { {"insert:AUTO1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, true, true, insertAUTOTest1, 10, 10, 2, 0, 0, 0, 1, -1}, {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryColumnTest, 10, 10, 1, 3, 0, 0, 1, 2}, - {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, + {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryMiscTest, 2, 10, 1, 3, 0, 0, 1, 2}, }; @@ -179,6 +191,8 @@ typedef struct { int32_t* bindTagTypeList; int32_t optrIdxListNum; int32_t* optrIdxList; + int32_t funcIdxListNum; + int32_t* funcIdxList; int32_t runTimes; int32_t caseIdx; // static case idx int32_t caseNum; // num in static case list @@ -186,7 +200,7 @@ typedef struct { int32_t caseRunNum; // total run case num } CaseCtrl; -#if 1 +#if 0 CaseCtrl gCaseCtrl = { // default .bindNullNum = 0, .printCreateTblSql = false, @@ -203,6 +217,8 @@ CaseCtrl gCaseCtrl = { // default .bindTagTypeList = NULL, .optrIdxListNum = 0, .optrIdxList = NULL, + .funcIdxListNum = 0, + .funcIdxList = NULL, .checkParamNum = false, .printRes = false, .runTimes = 0, @@ -241,7 +257,7 @@ CaseCtrl gCaseCtrl = { }; #endif -#if 0 +#if 1 CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindNullNum = 0, .printCreateTblSql = false, @@ -255,14 +271,14 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper .optrIdxListNum = 0, .optrIdxList = NULL, .checkParamNum = false, - .printRes = false, + .printRes = true, .runTimes = 0, .caseRunIdx = -1, .optrIdxListNum = 0, .optrIdxList = NULL, .bindColTypeNum = 0, .bindColTypeList = NULL, - .caseIdx = 23, + .caseIdx = 24, .caseNum = 1, .caseRunNum = 1, }; @@ -513,11 +529,83 @@ void bpAppendOperatorParam(BindData *data, int32_t *len, int32_t dataType, int32 } break; default: - printf("!!!invalid paramNum:%d\n", pInfo->paramNum); + printf("!!!invalid operator paramNum:%d\n", pInfo->paramNum); exit(1); } } +void bpAppendFunctionParam(BindData *data, int32_t *len, int32_t dataType, int32_t idx) { + FuncInfo *pInfo = NULL; + + if (gCaseCtrl.funcIdxListNum > 0) { + pInfo = &funcInfo[gCaseCtrl.funcIdxList[idx]]; + } else { + pInfo = &funcInfo[rand() % tListLen(funcInfo)]; + } + + switch (pInfo->paramNum) { + case 1: + *len += sprintf(data->sql + *len, " %s(?)", pInfo->funcName); + break; + default: + printf("!!!invalid function paramNum:%d\n", pInfo->paramNum); + exit(1); + } +} + + +int32_t bpAppendColumnName(BindData *data, int32_t type, int32_t len) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + return sprintf(data->sql + len, "booldata"); + break; + case TSDB_DATA_TYPE_TINYINT: + return sprintf(data->sql + len, "tinydata"); + break; + case TSDB_DATA_TYPE_SMALLINT: + return sprintf(data->sql + len, "smalldata"); + break; + case TSDB_DATA_TYPE_INT: + return sprintf(data->sql + len, "intdata"); + break; + case TSDB_DATA_TYPE_BIGINT: + return sprintf(data->sql + len, "bigdata"); + break; + case TSDB_DATA_TYPE_FLOAT: + return sprintf(data->sql + len, "floatdata"); + break; + case TSDB_DATA_TYPE_DOUBLE: + return sprintf(data->sql + len, "doubledata"); + break; + case TSDB_DATA_TYPE_VARCHAR: + return sprintf(data->sql + len, "binarydata"); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + return sprintf(data->sql + len, "ts"); + break; + case TSDB_DATA_TYPE_NCHAR: + return sprintf(data->sql + len, "nchardata"); + break; + case TSDB_DATA_TYPE_UTINYINT: + return sprintf(data->sql + len, "utinydata"); + break; + case TSDB_DATA_TYPE_USMALLINT: + return sprintf(data->sql + len, "usmalldata"); + break; + case TSDB_DATA_TYPE_UINT: + return sprintf(data->sql + len, "uintdata"); + break; + case TSDB_DATA_TYPE_UBIGINT: + return sprintf(data->sql + len, "ubigdata"); + break; + default: + printf("!!!invalid col type:%d", type); + exit(1); + } + + return 0; +} + void generateQueryCondSQL(BindData *data, int32_t tblIdx) { int32_t len = sprintf(data->sql, "select * from %s%d where ", bpTbPrefix, tblIdx); if (!gCurCase->fullCol) { @@ -525,53 +613,7 @@ void generateQueryCondSQL(BindData *data, int32_t tblIdx) { if (c) { len += sprintf(data->sql + len, " and "); } - switch (data->pBind[c].buffer_type) { - case TSDB_DATA_TYPE_BOOL: - len += sprintf(data->sql + len, "booldata"); - break; - case TSDB_DATA_TYPE_TINYINT: - len += sprintf(data->sql + len, "tinydata"); - break; - case TSDB_DATA_TYPE_SMALLINT: - len += sprintf(data->sql + len, "smalldata"); - break; - case TSDB_DATA_TYPE_INT: - len += sprintf(data->sql + len, "intdata"); - break; - case TSDB_DATA_TYPE_BIGINT: - len += sprintf(data->sql + len, "bigdata"); - break; - case TSDB_DATA_TYPE_FLOAT: - len += sprintf(data->sql + len, "floatdata"); - break; - case TSDB_DATA_TYPE_DOUBLE: - len += sprintf(data->sql + len, "doubledata"); - break; - case TSDB_DATA_TYPE_VARCHAR: - len += sprintf(data->sql + len, "binarydata"); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - len += sprintf(data->sql + len, "ts"); - break; - case TSDB_DATA_TYPE_NCHAR: - len += sprintf(data->sql + len, "nchardata"); - break; - case TSDB_DATA_TYPE_UTINYINT: - len += sprintf(data->sql + len, "utinydata"); - break; - case TSDB_DATA_TYPE_USMALLINT: - len += sprintf(data->sql + len, "usmalldata"); - break; - case TSDB_DATA_TYPE_UINT: - len += sprintf(data->sql + len, "uintdata"); - break; - case TSDB_DATA_TYPE_UBIGINT: - len += sprintf(data->sql + len, "ubigdata"); - break; - default: - printf("!!!invalid col type:%d", data->pBind[c].buffer_type); - exit(1); - } + len += bpAppendColumnName(data, data->pBind[c].buffer_type, len); bpAppendOperatorParam(data, &len, data->pBind[c].buffer_type, c); } @@ -582,64 +624,50 @@ void generateQueryCondSQL(BindData *data, int32_t tblIdx) { } } +void bpGenerateConstInOpSQL(BindData *data, int32_t tblIdx) { + int32_t len = 0; + len = sprintf(data->sql, "select "); + + for (int c = 0; c < gCurCase->bindColNum; ++c) { + if (c) { + len += sprintf(data->sql + len, ", "); + } + + len += bpAppendColumnName(data, data->pBind[c].buffer_type, len); + + bpAppendOperatorParam(data, &len, data->pBind[c].buffer_type, c); + } + + len += sprintf(data->sql + len, " from %s%d", bpTbPrefix, tblIdx); +} + + +void bpGenerateConstInFuncSQL(BindData *data, int32_t tblIdx) { + int32_t len = 0; + len = sprintf(data->sql, "select "); + + for (int c = 0; c < gCurCase->bindColNum; ++c) { + if (c) { + len += sprintf(data->sql + len, ", "); + } + + bpAppendFunctionParam(data, &len, data->pBind[c].buffer_type, c); + } + + len += sprintf(data->sql + len, " from %s%d", bpTbPrefix, tblIdx); +} + void generateQueryMiscSQL(BindData *data, int32_t tblIdx) { - int32_t len = sprintf(data->sql, "select * from %s%d where ", bpTbPrefix, tblIdx); - if (!gCurCase->fullCol) { - for (int c = 0; c < gCurCase->bindColNum; ++c) { - if (c) { - len += sprintf(data->sql + len, " and "); - } - switch (data->pBind[c].buffer_type) { - case TSDB_DATA_TYPE_BOOL: - len += sprintf(data->sql + len, "booldata"); - break; - case TSDB_DATA_TYPE_TINYINT: - len += sprintf(data->sql + len, "tinydata"); - break; - case TSDB_DATA_TYPE_SMALLINT: - len += sprintf(data->sql + len, "smalldata"); - break; - case TSDB_DATA_TYPE_INT: - len += sprintf(data->sql + len, "intdata"); - break; - case TSDB_DATA_TYPE_BIGINT: - len += sprintf(data->sql + len, "bigdata"); - break; - case TSDB_DATA_TYPE_FLOAT: - len += sprintf(data->sql + len, "floatdata"); - break; - case TSDB_DATA_TYPE_DOUBLE: - len += sprintf(data->sql + len, "doubledata"); - break; - case TSDB_DATA_TYPE_VARCHAR: - len += sprintf(data->sql + len, "binarydata"); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - len += sprintf(data->sql + len, "ts"); - break; - case TSDB_DATA_TYPE_NCHAR: - len += sprintf(data->sql + len, "nchardata"); - break; - case TSDB_DATA_TYPE_UTINYINT: - len += sprintf(data->sql + len, "utinydata"); - break; - case TSDB_DATA_TYPE_USMALLINT: - len += sprintf(data->sql + len, "usmalldata"); - break; - case TSDB_DATA_TYPE_UINT: - len += sprintf(data->sql + len, "uintdata"); - break; - case TSDB_DATA_TYPE_UBIGINT: - len += sprintf(data->sql + len, "ubigdata"); - break; - default: - printf("!!!invalid col type:%d", data->pBind[c].buffer_type); - exit(1); - } - - bpAppendOperatorParam(data, &len, data->pBind[c].buffer_type, c); - } + switch(tblIdx) { + case 0: + bpGenerateConstInOpSQL(data, tblIdx); + break; + case 1: + //TODO FILL TEST + default: + bpGenerateConstInFuncSQL(data, tblIdx); + break; } if (gCaseCtrl.printStmtSql) { From 7ed7e1de40c0005a90d833008417b86a6964af27 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 12 May 2022 11:21:44 +0000 Subject: [PATCH 86/94] feat: tag read --- include/common/tdataformat.h | 2 +- source/dnode/vnode/inc/vnode.h | 9 +++++---- source/dnode/vnode/src/meta/metaQuery.c | 5 +++++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 2e8a214a9d..c80b8db58a 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -384,7 +384,7 @@ static FORCE_INLINE int32_t comparTagId(const void *key1, const void *key2) { } } -static FORCE_INLINE void *tdGetKVRowValOfCol(SKVRow row, int16_t colId) { +static FORCE_INLINE void *tdGetKVRowValOfCol(const SKVRow row, int16_t colId) { void *ret = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ); if (ret == NULL) return NULL; return kvRowColVal(row, (SColIdx *)ret); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index ebf49c644b..b84732c848 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -72,10 +72,11 @@ typedef struct SMeta SMeta; // todo: remove typedef struct SMetaReader SMetaReader; typedef struct SMetaEntry SMetaEntry; -void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); -void metaReaderClear(SMetaReader *pReader); -int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); -int metaReadNext(SMetaReader *pReader); +void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); +void metaReaderClear(SMetaReader *pReader); +int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); +int metaReadNext(SMetaReader *pReader); +const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t cid); #if 1 // refact APIs below (TODO) typedef SVCreateTbReq STbCfg; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 7d0a87d79d..1e2c94679f 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -466,3 +466,8 @@ void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) { } #endif + +const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t cid) { + ASSERT(pEntry->type == TSDB_CHILD_TABLE); + return tdGetKVRowValOfCol((const SKVRow)pEntry->ctbEntry.pTags, cid); +} \ No newline at end of file From d0c284d18310e256b515bf96f1800ee64a56913e Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 12 May 2022 19:31:33 +0800 Subject: [PATCH 87/94] test: update test case --- tests/system-test/99-TDcase/TD-15517.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/99-TDcase/TD-15517.py b/tests/system-test/99-TDcase/TD-15517.py index e7837ba009..44a64421d3 100644 --- a/tests/system-test/99-TDcase/TD-15517.py +++ b/tests/system-test/99-TDcase/TD-15517.py @@ -251,7 +251,7 @@ class TDTestCase: # wait db ready while 1: tdSql.query("show databases") - if tdSql.getRows() == 3: + if tdSql.getRows() == 4: print (tdSql.getData(0,0), tdSql.getData(1,0),tdSql.getData(2,0),) break else: From d2cfe7a8e0453d7550ef9d06205b9cadaf4e04f7 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 12 May 2022 19:39:40 +0800 Subject: [PATCH 88/94] test: update test case --- tests/system-test/99-TDcase/TD-15517.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/99-TDcase/TD-15517.py b/tests/system-test/99-TDcase/TD-15517.py index 44a64421d3..b7cac43954 100644 --- a/tests/system-test/99-TDcase/TD-15517.py +++ b/tests/system-test/99-TDcase/TD-15517.py @@ -131,7 +131,7 @@ class TDTestCase: 'vgroups': 1, \ 'stbName': 'stb', \ 'ctbNum': 10, \ - 'rowsPerTbl': 10000, \ + 'rowsPerTbl': 100, \ 'batchNum': 10, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath From 64a54c15419d63a8a3220d6aee73d3cf3ec5d865 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 12 May 2022 19:57:17 +0800 Subject: [PATCH 89/94] fix bug --- source/libs/scalar/src/filter.c | 5 +++++ source/libs/scalar/src/scalar.c | 1 + 2 files changed, 6 insertions(+) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 91ecef6161..4317ad325e 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3601,6 +3601,11 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } + if (FILTER_GET_FLAG(stat->info->options, FLT_OPTION_TIMESTAMP) && node->opType >= OP_TYPE_NOT_EQUAL) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + if (NULL == node->pRight) { if (scalarGetOperatorParamNum(node->opType) > 1) { fltError("invalid operator, pRight:%p, nodeType:%d, opType:%d", node->pRight, nodeType(node), node->opType); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index c63f0a27db..d7cac69d26 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -8,6 +8,7 @@ #include "tdatablock.h" #include "scalar.h" #include "tudf.h" +#include "ttime.h" int32_t scalarGetOperatorParamNum(EOperatorType type) { if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type From b57f62f139919c6521e3494ae151b3e3c6624987 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 12 May 2022 20:33:52 +0800 Subject: [PATCH 90/94] feat: adjust query stmt APIs --- include/libs/nodes/querynodes.h | 1 + include/libs/parser/parser.h | 18 ++-- source/client/src/clientImpl.c | 6 +- source/libs/parser/inc/parAst.h | 1 + source/libs/parser/src/parAstCreater.c | 8 ++ source/libs/parser/src/parAstParser.c | 2 + source/libs/parser/src/parser.c | 115 +++++++++++++++++++++- source/libs/planner/test/planTestUtil.cpp | 2 +- 8 files changed, 138 insertions(+), 15 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index d07f29c487..8232dbba0f 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -324,6 +324,7 @@ typedef struct SQuery { SArray* pTableList; bool showRewrite; int32_t placeholderNum; + SArray* pPlaceholderValues; } SQuery; void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 61e4bb3723..ca399a0235 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -47,21 +47,23 @@ typedef struct SParseContext { bool isSuperUser; } SParseContext; -int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery); +int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery); bool isInsertSql(const char* pStr, size_t length); void qDestroyQuery(SQuery* pQueryNode); int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); -int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash); -int32_t qResetStmtDataBlock(void* block, bool keepBuf); -int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); -void qFreeStmtDataBlock(void* pDataBlock); -int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId); -void qDestroyStmtDataBlock(void* pBlock); -STableMeta *qGetTableMetaInDataBlock(void* pDataBlock); +int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash); +int32_t qResetStmtDataBlock(void* block, bool keepBuf); +int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); +void qFreeStmtDataBlock(void* pDataBlock); +int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId); +void qDestroyStmtDataBlock(void* pBlock); +STableMeta* qGetTableMetaInDataBlock(void* pDataBlock); +int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId); +int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery); int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index cb361f09a6..f879838d63 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -180,7 +180,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC return code; } - code = qParseQuerySql(&cxt, pQuery); + code = qParseSql(&cxt, pQuery); if (TSDB_CODE_SUCCESS == code) { if ((*pQuery)->haveResultSet) { setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols); @@ -554,8 +554,8 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { } taosMemoryFreeClear(db); - connectReq.connType = pObj->connType; - connectReq.pid = htonl(appInfo.pid); + connectReq.connType = pObj->connType; + connectReq.pid = htonl(appInfo.pid); connectReq.startTime = htobe64(appInfo.startTime); tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app)); diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index b15078157a..fc096a057c 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -32,6 +32,7 @@ typedef struct SAstCreateContext { bool notSupport; SNode* pRootNode; int16_t placeholderNo; + SArray* pPlaceholderValues; int32_t errCode; } SAstCreateContext; diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 74a023f0cd..b0930f69d4 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -299,6 +299,14 @@ SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLitera val->literal = strndup(pLiteral->z, pLiteral->n); CHECK_OUT_OF_MEM(val->literal); val->placeholderNo = ++pCxt->placeholderNo; + if (NULL == pCxt->pPlaceholderValues) { + pCxt->pPlaceholderValues = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + if (NULL == pCxt->pPlaceholderValues) { + nodesDestroyNode(val); + return NULL; + } + } + taosArrayPush(pCxt->pPlaceholderValues, &val); return (SNode*)val; } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index ebc8281f56..5b59d1c080 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -81,6 +81,8 @@ abort_parse: } (*pQuery)->pRoot = cxt.pRootNode; (*pQuery)->placeholderNum = cxt.placeholderNo; + TSWAP((*pQuery)->pPlaceholderValues, cxt.pPlaceholderValues); } + taosArrayDestroy(cxt.pPlaceholderValues); return cxt.errCode; } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index bc98a41923..2652078b96 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -39,16 +39,99 @@ static int32_t parseSqlIntoAst(SParseContext* pCxt, SQuery** pQuery) { if (TSDB_CODE_SUCCESS == code) { code = authenticate(pCxt, *pQuery); } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && 0 == (*pQuery)->placeholderNum) { code = translate(pCxt, *pQuery); } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && 0 == (*pQuery)->placeholderNum) { code = calculateConstant(pCxt, *pQuery); } return code; } -int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery) { +static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { + if (pParam->is_null && 1 == *(pParam->is_null)) { + pVal->node.resType.type = TSDB_DATA_TYPE_NULL; + pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_NULL].bytes; + return TSDB_CODE_SUCCESS; + } + int32_t inputSize = (NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes); + pVal->node.resType.type = pParam->buffer_type; + pVal->node.resType.bytes = inputSize; + switch (pParam->buffer_type) { + case TSDB_DATA_TYPE_BOOL: + pVal->datum.b = *((bool*)pParam->buffer); + break; + case TSDB_DATA_TYPE_TINYINT: + pVal->datum.i = *((int8_t*)pParam->buffer); + break; + case TSDB_DATA_TYPE_SMALLINT: + pVal->datum.i = *((int16_t*)pParam->buffer); + break; + case TSDB_DATA_TYPE_INT: + pVal->datum.i = *((int32_t*)pParam->buffer); + break; + case TSDB_DATA_TYPE_BIGINT: + pVal->datum.i = *((int64_t*)pParam->buffer); + break; + case TSDB_DATA_TYPE_FLOAT: + pVal->datum.d = *((float*)pParam->buffer); + break; + case TSDB_DATA_TYPE_DOUBLE: + pVal->datum.d = *((double*)pParam->buffer); + break; + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: + pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1); + if (NULL == pVal->datum.p) { + return TSDB_CODE_OUT_OF_MEMORY; + } + varDataSetLen(pVal->datum.p, pVal->node.resType.bytes); + strncpy(varDataVal(pVal->datum.p), (const char*)pParam->buffer, pVal->node.resType.bytes); + break; + case TSDB_DATA_TYPE_NCHAR: { + pVal->node.resType.bytes *= TSDB_NCHAR_SIZE; + pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1); + if (NULL == pVal->datum.p) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t output = 0; + if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes, + &output)) { + return errno; + } + varDataSetLen(pVal->datum.p, output); + pVal->node.resType.bytes = output; + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: + pVal->datum.i = *((int64_t*)pParam->buffer); + break; + case TSDB_DATA_TYPE_UTINYINT: + pVal->datum.u = *((uint8_t*)pParam->buffer); + break; + case TSDB_DATA_TYPE_USMALLINT: + pVal->datum.u = *((uint16_t*)pParam->buffer); + break; + case TSDB_DATA_TYPE_UINT: + pVal->datum.u = *((uint32_t*)pParam->buffer); + break; + case TSDB_DATA_TYPE_UBIGINT: + pVal->datum.u = *((uint64_t*)pParam->buffer); + break; + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_MEDIUMBLOB: + // todo + default: + break; + } + pVal->translate = true; + return TSDB_CODE_SUCCESS; +} + +int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery) { int32_t code = TSDB_CODE_SUCCESS; if (isInsertSql(pCxt->pSql, pCxt->sqlLen)) { code = parseInsertSql(pCxt, pQuery); @@ -77,3 +160,29 @@ void qDestroyQuery(SQuery* pQueryNode) { int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) { return extractResultSchema(pRoot, numOfCols, pSchema); } + +int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId) { + int32_t code = TSDB_CODE_SUCCESS; + + if (colIdx < 0) { + int32_t size = taosArrayGetSize(pQuery->pPlaceholderValues); + for (int32_t i = 0; i < size; ++i) { + code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + } else { + code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams); + } + + return code; +} + +int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery) { + int32_t code = translate(pCxt, pQuery); + if (TSDB_CODE_SUCCESS == code) { + code = calculateConstant(pCxt, pQuery); + } + return code; +} diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index e434a6be96..b2c590667e 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -198,7 +198,7 @@ class PlannerTestBaseImpl { cxt.pMsg = stmtEnv_.msgBuf_.data(); cxt.msgLen = stmtEnv_.msgBuf_.max_size(); - DO_WITH_THROW(qParseQuerySql, &cxt, pQuery); + DO_WITH_THROW(qParseSql, &cxt, pQuery); res_.ast_ = toString((*pQuery)->pRoot); } From 1c98d780a074052b3c355f8b7da0aac5c35b6842 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 12 May 2022 20:51:02 +0800 Subject: [PATCH 91/94] fix(wal): int overflow --- source/dnode/mnode/impl/src/mndTopic.c | 2 ++ source/libs/wal/src/walRead.c | 4 +-- source/os/src/osFile.c | 36 +++++++------------------- 3 files changed, 13 insertions(+), 29 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 00379ecda1..bd2923ac1a 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -485,8 +485,10 @@ static int32_t mndProcessDropTopicReq(SNodeMsg *pReq) { return -1; } } + // TODO: check ref int32_t code = mndDropTopic(pMnode, pReq, pTopic); + // TODO: iterate and drop related subscriptions and offsets mndReleaseTopic(pMnode, pTopic); if (code != 0) { diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index b04dea0213..38c14895d8 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -55,7 +55,7 @@ int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { } static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { - int ret = 0; + int64_t ret = 0; TdFilePtr pIdxTFile = pRead->pReadIdxTFile; TdFilePtr pLogTFile = pRead->pReadLogTFile; @@ -68,7 +68,7 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i wError("failed to seek idx file, ver %ld, pos: %ld, since %s", ver, offset, terrstr()); return -1; } - SWalIdxEntry entry; + SWalIdxEntry entry = {0}; if ((ret = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry))) != sizeof(SWalIdxEntry)) { if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 4fd672fe3c..425bf8b7ac 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -246,11 +246,11 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0; access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0; access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0; - #ifdef WINDOWS - fd = _open(path, access, _S_IREAD|_S_IWRITE); - #else +#ifdef WINDOWS + fd = _open(path, access, _S_IREAD | _S_IWRITE); +#else fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); - #endif +#endif if (fd == -1) { return NULL; } @@ -310,9 +310,6 @@ int64_t taosCloseFile(TdFilePtr *ppFile) { } int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { - if (pFile == NULL) { - return 0; - } #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif @@ -356,10 +353,10 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. #ifdef WINDOWS size_t pos = lseek(pFile->fd, 0, SEEK_CUR); - lseek(pFile->fd, (long)offset, SEEK_SET); + lseek(pFile->fd, offset, SEEK_SET); int64_t ret = read(pFile->fd, buf, count); lseek(pFile->fd, pos, SEEK_SET); #else @@ -372,9 +369,6 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) } int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { - if (pFile == NULL) { - return 0; - } #if FILE_WITH_LOCK taosThreadRwlockWrlock(&(pFile->rwlock)); #endif @@ -406,14 +400,11 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { } int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { - if (pFile == NULL) { - return 0; - } #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif assert(pFile->fd >= 0); // Please check if you have closed the file. - int64_t ret = lseek(pFile->fd, (long)offset, whence); + int64_t ret = lseek(pFile->fd, offset, whence); #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); #endif @@ -424,9 +415,6 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { #ifdef WINDOWS return 0; #else - if (pFile == NULL) { - return 0; - } assert(pFile->fd >= 0); // Please check if you have closed the file. struct stat fileStat; @@ -451,9 +439,6 @@ int32_t taosLockFile(TdFilePtr pFile) { #ifdef WINDOWS return 0; #else - if (pFile == NULL) { - return 0; - } assert(pFile->fd >= 0); // Please check if you have closed the file. return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); @@ -464,9 +449,6 @@ int32_t taosUnLockFile(TdFilePtr pFile) { #ifdef WINDOWS return 0; #else - if (pFile == NULL) { - return 0; - } assert(pFile->fd >= 0); // Please check if you have closed the file. return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); @@ -667,7 +649,7 @@ int32_t taosUmaskFile(int32_t maskVal) { int32_t taosGetErrorFile(TdFilePtr pFile) { return errno; } int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { - if (pFile == NULL || ptrBuf == NULL ) { + if (pFile == NULL || ptrBuf == NULL) { return -1; } if (*ptrBuf != NULL) { @@ -689,7 +671,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { #endif } int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) { - if (pFile == NULL || buf == NULL ) { + if (pFile == NULL || buf == NULL) { return -1; } assert(pFile->fp != NULL); From c4c81c83af76657bdf5cd9b95ebc0d2deceade77 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 12 May 2022 20:53:52 +0800 Subject: [PATCH 92/94] [test: modify case for tmq] --- tests/system-test/7-tmq/basic5.py | 177 ++++++++++++++++++++++++------ 1 file changed, 145 insertions(+), 32 deletions(-) diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py index 267b6ff5d8..8a1932f05c 100644 --- a/tests/system-test/7-tmq/basic5.py +++ b/tests/system-test/7-tmq/basic5.py @@ -110,19 +110,10 @@ class TDTestCase: parameterDict["rowsPerTbl"],\ parameterDict["batchNum"],\ parameterDict["startTs"]) - return - - def run(self): - tdSql.prepare() + return - buildPath = self.getBuildPath() - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - cfgPath = buildPath + "/../sim/psim/cfg" - tdLog.info("cfgPath: %s" % cfgPath) + def tmqCase1(self, cfgPath, buildPath): tdLog.printNoPrefix("======== test scenario 1: ") tdLog.info("step 1: create database, stb, ctb and insert data") # create and start thread @@ -131,7 +122,7 @@ class TDTestCase: 'vgroups': 1, \ 'stbName': 'stb', \ 'ctbNum': 10, \ - 'rowsPerTbl': 10000, \ + 'rowsPerTbl': 100, \ 'batchNum': 10, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath @@ -141,10 +132,7 @@ class TDTestCase: # wait stb ready while 1: - #tdSql.query("show %s.stables"%parameterDict['dbName']) - tdSql.query("show db.stables") - #print (self.queryResult) - #print (tdSql.getRows()) + tdSql.query("show %s.stables"%parameterDict['dbName']) if tdSql.getRows() == 1: break else: @@ -159,26 +147,18 @@ class TDTestCase: time.sleep(1) tdSql.query("show topics") - print ("======================================") - #print (self.queryResult) #tdSql.checkRows(2) topic1 = tdSql.getData(0 , 0) topic2 = tdSql.getData(1 , 0) - print (topic1) - print (topic2) - print (topicFromStb) - print (topicFromCtb) - #tdLog.info("show topics: %s, %s"%topic1, topic2) - #if topic1 != topicFromStb or topic1 != topicFromCtb: - # tdLog.exit("topic error1") - #if topic2 != topicFromStb or topic2 != topicFromCtb: - # tdLog.exit("topic error2") + tdLog.info("show topics: %s, %s"%(topic1, topic2)) + if topic1 != topicFromStb and topic1 != topicFromCtb: + tdLog.exit("topic error1") + if topic2 != topicFromStb and topic2 != topicFromCtb: + tdLog.exit("topic error2") tdLog.info("create consume info table and consume result table") cdbName = parameterDict["dbName"] - #tdSql.query("create database %s"%cdbName) - #tdSql.query("use %s"%cdbName) tdSql.query("create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)") tdSql.query("create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)") @@ -234,14 +214,147 @@ class TDTestCase: tdSql.checkData(0 , 1, consumerId) tdSql.checkData(0 , 2, expectmsgcnt) tdSql.checkData(0 , 3, expectrowcnt) + + tdSql.query("drop topic %s"%topicFromStb) + tdSql.query("drop topic %s"%topicFromCtb) - tdLog.printNoPrefix("======== test scenario 2: ") + + def tmqCase2(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test scenario 2: add child table with consuming ") + # create and start thread + parameterDict = {'cfg': '', \ + 'dbName': 'db2', \ + 'vgroups': 1, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + # wait db ready + while 1: + tdSql.query("show databases") + if tdSql.getRows() == 4: + print (tdSql.getData(0,0), tdSql.getData(1,0),tdSql.getData(2,0),) + break + else: + time.sleep(1) + + tdSql.query("use %s"%parameterDict['dbName']) + # wait stb ready + while 1: + tdSql.query("show %s.stables"%parameterDict['dbName']) + if tdSql.getRows() == 1: + break + else: + time.sleep(1) - tdLog.printNoPrefix("======== test scenario 3: ") + tdLog.info("create topics from super table") + topicFromStb = 'topic_stb_column2' + topicFromCtb = 'topic_ctb_column2' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName'])) + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName'])) + + time.sleep(1) + tdSql.query("show topics") + topic1 = tdSql.getData(0 , 0) + topic2 = tdSql.getData(1 , 0) + tdLog.info("show topics: %s, %s"%(topic1, topic2)) + if topic1 != topicFromStb and topic1 != topicFromCtb: + tdLog.exit("topic error1") + if topic2 != topicFromStb and topic2 != topicFromCtb: + tdLog.exit("topic error2") + + tdLog.info("create consume info table and consume result table") + cdbName = parameterDict["dbName"] + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) - #os.system('pkill tmq_sim') + consumerId = 0 + expectmsgcnt = (parameterDict["rowsPerTbl"] / parameterDict["batchNum"] ) * parameterDict["ctbNum"] + expectmsgcnt1 = expectmsgcnt + parameterDict["ctbNum"] + topicList = topicFromStb + ifcheckdata = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + sql = "insert into consumeinfo values " + sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectmsgcnt1, ifcheckdata) + tdSql.query(sql) + + tdLog.info("check stb if there are data") + while 1: + tdSql.query("select count(*) from %s"%parameterDict["stbName"]) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + countOfStb = tdSql.getData(0, 0) + if countOfStb != 0: + tdLog.info("count from stb: %d"%countOfStb) + break + else: + time.sleep(1) + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" + tdLog.info(shellCmd) + os.system(shellCmd) + # create new child table and insert data + newCtbName = 'newctb' + rowsOfNewCtb = 1000 + tdSql.query("create table %s.%s using %s.%s tags(9999)"%(parameterDict["dbName"], newCtbName, parameterDict["dbName"], parameterDict["stbName"])) + startTs = parameterDict["startTs"] + for j in range(rowsOfNewCtb): + sql = "insert into %s.%s values (%d, %d, 'tmqrow_%d') "%(parameterDict["dbName"], newCtbName, startTs + j, j, j) + tdSql.execute(sql) + tdLog.debug("insert data into new child table ............ [OK]") + + # wait for data ready + prepareEnvThread.join() + + tdLog.info("insert process end, and start to check consume result") + while 1: + tdSql.query("select * from consumeresult") + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == 1: + break + else: + time.sleep(5) + + expectmsgcnt += rowsOfNewCtb + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + rowsOfNewCtb + + tdSql.checkData(0 , 1, consumerId) + tdSql.checkData(0 , 2, expectmsgcnt) + tdSql.checkData(0 , 3, expectrowcnt) + + tdLog.printNoPrefix("======== test scenario 2 end ...... ") + + def run(self): + tdSql.prepare() + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + self.tmqCase1(cfgPath, buildPath) + #self.tmqCase2(cfgPath, buildPath) + #self.tmqCase3(cfgPath, buildPath) def stop(self): tdSql.close() From af2d2db5d3d158042520aa38d511e0f7135ba3d0 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 12 May 2022 20:56:29 +0800 Subject: [PATCH 93/94] fix(wal): int overflow --- source/libs/wal/src/walRead.c | 4 ++-- source/libs/wal/src/walWrite.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 38c14895d8..a0c65c8ed6 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -75,7 +75,7 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i wError("failed to read idx file, since %s", terrstr()); } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - wError("read idx file incompletely, read bytes %d, bytes should be %lu", ret, sizeof(SWalIdxEntry)); + wError("read idx file incompletely, read bytes %ld, bytes should be %lu", ret, sizeof(SWalIdxEntry)); } return -1; } @@ -187,7 +187,7 @@ int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead) { } int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalHead *pHead) { - int32_t code; + int64_t code; ASSERT(pRead->curVersion == pHead->head.version); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 2e43997584..2ddcb27835 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -30,8 +30,8 @@ int32_t walCommit(SWal *pWal, int64_t ver) { } int32_t walRollback(SWal *pWal, int64_t ver) { - int code; - char fnameStr[WAL_FILE_LEN]; + int64_t code; + char fnameStr[WAL_FILE_LEN]; if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { terrno = TSDB_CODE_WAL_INVALID_VER; return -1; From 04a2368f6d88c5f34424e8dcb31cdcf8786b3302 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 12 May 2022 20:57:34 +0800 Subject: [PATCH 94/94] feat: adjust query stmt APIs --- source/libs/parser/src/parAstCreater.c | 1 + source/libs/planner/test/planStmtTest.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index b0930f69d4..639da98f48 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -44,6 +44,7 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) { pCxt->notSupport = false; pCxt->pRootNode = NULL; pCxt->placeholderNo = 0; + pCxt->pPlaceholderValues = NULL; pCxt->errCode = TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/test/planStmtTest.cpp b/source/libs/planner/test/planStmtTest.cpp index 6d56a3419e..0674a69355 100644 --- a/source/libs/planner/test/planStmtTest.cpp +++ b/source/libs/planner/test/planStmtTest.cpp @@ -50,5 +50,5 @@ class PlanStmtTest : public PlannerTestBase { TEST_F(PlanStmtTest, stmt) { useDb("root", "test"); - run("select * from t1 where c1 = ?"); + // run("select * from t1 where c1 = ?"); }