From df76d0ade29c03100322c56c121d7c43882328bd Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 2 Sep 2022 15:34:44 +0800 Subject: [PATCH 01/16] fix: rlock for meta uid index querying from query related threads --- source/dnode/vnode/src/meta/metaQuery.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 7df355a59b..9cbacd4e36 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -129,10 +129,16 @@ _err: bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) { // query uid.idx + metaRLock(pMeta); + if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) { + metaULock(pMeta); + return false; } + metaULock(pMeta); + return true; } @@ -182,9 +188,14 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) { } int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName) { + int code = 0; SMetaReader mr = {0}; metaReaderInit(&mr, (SMeta *)meta, 0); - metaGetTableEntryByUid(&mr, uid); + code = metaGetTableEntryByUid(&mr, uid); + if (code < 0) { + metaReaderClear(&mr); + return -1; + } STR_TO_VARSTR(tbName, mr.me.name); metaReaderClear(&mr); From d653da7971bab0a9bc655d9c243125457a100ce2 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 2 Sep 2022 15:34:16 +0800 Subject: [PATCH 02/16] feat(stream): support delete data --- include/common/tdatablock.h | 11 +- source/common/src/tdatablock.c | 87 ++- source/dnode/vnode/src/inc/vnodeInt.h | 5 +- source/dnode/vnode/src/tq/tq.c | 80 +- source/dnode/vnode/src/tq/tqPush.c | 29 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +- source/libs/executor/inc/executorimpl.h | 704 +++++++++--------- source/libs/executor/src/scanoperator.c | 41 +- source/libs/executor/src/timewindowoperator.c | 174 ++--- source/libs/stream/src/streamDispatch.c | 54 +- 10 files changed, 665 insertions(+), 526 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 410fa02ded..73d043b2d0 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -184,7 +184,8 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u int32_t getJsonValueLen(const char* data); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); -int32_t colDataAppendNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t numOfRows); +int32_t colDataAppendNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, + uint32_t numOfRows); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, int32_t numOfRow2); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, @@ -225,15 +226,16 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n); int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n); -int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src); -int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src); +int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src); +int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src); SSDataBlock* createDataBlock(); void* blockDataDestroy(SSDataBlock* pBlock); void blockDataFreeRes(SSDataBlock* pBlock); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); +SSDataBlock* createSpecialDataBlock(EStreamType type); -int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData); +int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData); SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId); SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); @@ -249,7 +251,6 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf); int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataBlocks, STSchema* pTSchema, int32_t vgId, tb_uid_t suid); - char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index c7f372f17b..bcb96dafcc 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -140,7 +140,8 @@ int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize) { return TSDB_CODE_SUCCESS; } -static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t currentRow, const char* pData, int32_t itemLen, int32_t numOfRows) { +static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t currentRow, const char* pData, + int32_t itemLen, int32_t numOfRows) { ASSERT(pColumnInfoData->info.bytes >= itemLen); size_t start = 1; @@ -148,21 +149,23 @@ static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t curren memcpy(pColumnInfoData->pData, pData, itemLen); int32_t t = 0; - int32_t count = log(numOfRows)/log(2); - while(t < count) { + int32_t count = log(numOfRows) / log(2); + while (t < count) { int32_t xlen = 1 << t; - memcpy(pColumnInfoData->pData + start * itemLen + pColumnInfoData->varmeta.length, pColumnInfoData->pData, xlen * itemLen); + memcpy(pColumnInfoData->pData + start * itemLen + pColumnInfoData->varmeta.length, pColumnInfoData->pData, + xlen * itemLen); t += 1; start += xlen; } // the tail part if (numOfRows > start) { - memcpy(pColumnInfoData->pData + start * itemLen + currentRow * itemLen, pColumnInfoData->pData, (numOfRows - start) * itemLen); + memcpy(pColumnInfoData->pData + start * itemLen + currentRow * itemLen, pColumnInfoData->pData, + (numOfRows - start) * itemLen); } if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - for(int32_t i = 0; i < numOfRows; ++i) { + for (int32_t i = 0; i < numOfRows; ++i) { pColumnInfoData->varmeta.offset[i + currentRow] = pColumnInfoData->varmeta.length + i * itemLen; } @@ -170,7 +173,8 @@ static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t curren } } -int32_t colDataAppendNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t numOfRows) { +int32_t colDataAppendNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, + uint32_t numOfRows) { ASSERT(pData != NULL && pColumnInfoData != NULL); int32_t len = pColumnInfoData->info.bytes; @@ -278,7 +282,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int } else { if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) { // all data may be null, when the pColumnInfoData->info.type == 0, bytes == 0; -// ASSERT(finalNumOfRows * pColumnInfoData->info.bytes); + // ASSERT(finalNumOfRows * pColumnInfoData->info.bytes); char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); if (tmp == NULL) { return TSDB_CODE_VND_OUT_OF_MEMORY; @@ -557,7 +561,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { } int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { - int32_t numOfRows = *(int32_t*) buf; + int32_t numOfRows = *(int32_t*)buf; blockDataEnsureCapacity(pBlock, numOfRows); pBlock->info.rows = numOfRows; @@ -676,7 +680,8 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { * @return */ size_t blockDataGetSerialMetaSize(uint32_t numOfCols) { - // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column + // length | return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int8_t) + sizeof(int32_t)) + numOfCols * sizeof(int32_t); } @@ -1302,6 +1307,40 @@ int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) { return TSDB_CODE_SUCCESS; } +SSDataBlock* createSpecialDataBlock(EStreamType type) { + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + pBlock->info.hasVarCol = false; + pBlock->info.groupId = 0; + pBlock->info.rows = 0; + pBlock->info.type = type; + pBlock->info.rowSize = + sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t) + sizeof(uint64_t) + sizeof(TSKEY) + sizeof(TSKEY); + pBlock->info.watermark = INT64_MIN; + + pBlock->pDataBlock = taosArrayInit(6, sizeof(SColumnInfoData)); + SColumnInfoData infoData = {0}; + infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP; + infoData.info.bytes = sizeof(TSKEY); + // window start ts + taosArrayPush(pBlock->pDataBlock, &infoData); + // window end ts + taosArrayPush(pBlock->pDataBlock, &infoData); + + infoData.info.type = TSDB_DATA_TYPE_UBIGINT; + infoData.info.bytes = sizeof(uint64_t); + // uid + taosArrayPush(pBlock->pDataBlock, &infoData); + // group id + taosArrayPush(pBlock->pDataBlock, &infoData); + + // calculate start ts + taosArrayPush(pBlock->pDataBlock, &infoData); + // calculate end ts + taosArrayPush(pBlock->pDataBlock, &infoData); + + return pBlock; +} + SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { if (pDataBlock == NULL) { return NULL; @@ -1426,7 +1465,7 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { } void colDataDestroy(SColumnInfoData* pColData) { - if(!pColData) return; + if (!pColData) return; if (IS_VAR_DATA_TYPE(pColData->info.type)) { taosMemoryFreeClear(pColData->varmeta.offset); } else { @@ -1693,7 +1732,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { } struct tm ptm = {0}; taosLocalTime(&tt, &ptm); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); if (precision == TSDB_TIME_PRECISION_NANO) { sprintf(buf + pos, ".%09d", ms); @@ -1847,20 +1886,20 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) break; case TSDB_DATA_TYPE_VARCHAR: { memset(pBuf, 0, sizeof(pBuf)); - char* pData = colDataGetVarData(pColInfoData, j); + char* pData = colDataGetVarData(pColInfoData, j); int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData)); memcpy(pBuf, varDataVal(pData), dataSize); len += snprintf(dumpBuf + len, size - len, " %15s |", pBuf); if (len >= size - 1) return dumpBuf; - } break; + } break; case TSDB_DATA_TYPE_NCHAR: { - char* pData = colDataGetVarData(pColInfoData, j); + char* pData = colDataGetVarData(pColInfoData, j); int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData)); memset(pBuf, 0, sizeof(pBuf)); - taosUcs4ToMbs((TdUcs4 *)varDataVal(pData), dataSize, pBuf); + taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); len += snprintf(dumpBuf + len, size - len, " %15s |", pBuf); if (len >= size - 1) return dumpBuf; - } break; + } break; } } len += snprintf(dumpBuf + len, size - len, "\n"); @@ -1877,7 +1916,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) * @param pDataBlocks * @param vgId * @param suid - * + * */ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataBlock, STSchema* pTSchema, int32_t vgId, tb_uid_t suid) { @@ -1904,8 +1943,8 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB tdSRowInit(&rb, pTSchema->version); for (int32_t i = 0; i < sz; ++i) { - int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); - int32_t rows = pDataBlock->info.rows; + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; // int32_t rowSize = pDataBlock->info.rowSize; // int64_t groupId = pDataBlock->info.groupId; @@ -1926,7 +1965,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB msgLen += sizeof(SSubmitBlk); int32_t dataLen = 0; - for (int32_t j = 0; j < rows; ++j) { // iterate by row + for (int32_t j = 0; j < rows; ++j) { // iterate by row tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen + dataLen)); // set row buf bool isStartKey = false; int32_t offset = 0; @@ -2089,7 +2128,7 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ // flag segment. // the inital bit is for column info int32_t* flagSegment = (int32_t*)data; - *flagSegment = (1<<31); + *flagSegment = (1 << 31); data += sizeof(int32_t); @@ -2149,7 +2188,7 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { const char* pStart = pData; - int32_t version = *(int32_t*) pStart; + int32_t version = *(int32_t*)pStart; pStart += sizeof(int32_t); ASSERT(version == 1); @@ -2158,7 +2197,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { pStart += sizeof(int32_t); // total rows sizeof(int32_t) - int32_t numOfRows = *(int32_t*)pStart; + int32_t numOfRows = *(int32_t*)pStart; pStart += sizeof(int32_t); // total columns sizeof(int32_t) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9b252df58b..c644f1d08c 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -102,7 +102,7 @@ int metaCommit(SMeta* pMeta); int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList); -int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp **pMetaRsp); +int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** pMetaRsp); int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids); int metaTtlDropTable(SMeta* pMeta, int64_t ttl, SArray* tbUids); int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp); @@ -173,7 +173,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); // tq-stream int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); -int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* data, int64_t ver); +int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* data, int64_t ver); +int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver); int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec); int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 3f42d8360e..f8fac97710 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -816,7 +816,85 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msg return streamMetaAddSerializedTask(pTq->pStreamMeta, version, msg, msgLen); } -int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq, int64_t ver) { +int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { + bool failed = false; + SDecoder* pCoder = &(SDecoder){0}; + SDeleteRes* pRes = &(SDeleteRes){0}; + + pRes->uidList = taosArrayInit(0, sizeof(tb_uid_t)); + if (pRes->uidList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + failed = true; + } + + tDecoderInit(pCoder, pReq, len); + tDecodeDeleteRes(pCoder, pRes); + /*ASSERT(pRes->skey != 0);*/ + /*ASSERT(pRes->ekey != 0);*/ + tDecoderClear(pCoder); + + int32_t sz = taosArrayGetSize(pRes->uidList); + if (sz == 0) { + taosArrayDestroy(pRes->uidList); + return 0; + } + SSDataBlock* pDelBlock = createSpecialDataBlock(STREAM_DELETE_DATA); + blockDataEnsureCapacity(pDelBlock, sz); + pDelBlock->info.rows = sz; + pDelBlock->info.version = ver; + + for (int32_t i = 0; i < sz; i++) { + // start key column + SColumnInfoData* pStartCol = taosArrayGet(pDelBlock->pDataBlock, START_TS_COLUMN_INDEX); + colDataAppend(pStartCol, i, (const char*)&pRes->skey, false); // end key column + SColumnInfoData* pEndCol = taosArrayGet(pDelBlock->pDataBlock, END_TS_COLUMN_INDEX); + colDataAppend(pEndCol, i, (const char*)&pRes->ekey, false); + // uid column + SColumnInfoData* pUidCol = taosArrayGet(pDelBlock->pDataBlock, UID_COLUMN_INDEX); + int64_t* pUid = taosArrayGet(pRes->uidList, i); + colDataAppend(pUidCol, i, (const char*)pUid, false); + + colDataAppendNULL(taosArrayGet(pDelBlock->pDataBlock, GROUPID_COLUMN_INDEX), i); + colDataAppendNULL(taosArrayGet(pDelBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX), i); + colDataAppendNULL(taosArrayGet(pDelBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX), i); + } + + void* pIter = NULL; + while (1) { + pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); + if (pIter == NULL) break; + SStreamTask* pTask = *(SStreamTask**)pIter; + if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue; + + qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); + + SStreamDataBlock* pStreamBlock = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); + pStreamBlock->type = STREAM_INPUT__DATA_BLOCK; + pStreamBlock->blocks = taosArrayInit(0, sizeof(SSDataBlock)); + SSDataBlock block = {0}; + assignOneDataBlock(&block, pDelBlock); + block.info.type = STREAM_DELETE_DATA; + taosArrayPush(pStreamBlock->blocks, &block); + + if (!failed) { + if (streamTaskInput(pTask, (SStreamQueueItem*)pStreamBlock) < 0) { + qError("stream task input del failed, task id %d", pTask->taskId); + continue; + } + + if (streamSchedExec(pTask) < 0) { + qError("stream task launch failed, task id %d", pTask->taskId); + continue; + } + } else { + streamTaskInputFail(pTask); + } + } + + return 0; +} + +int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) { void* pIter = NULL; bool failed = false; SStreamDataSubmit* pSubmit = NULL; diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index ed7fa80c47..a57e8174fe 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -213,20 +213,25 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ #endif int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { - if (vnodeIsRoleLeader(pTq->pVnode) && msgType == TDMT_VND_SUBMIT) { - if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0; + if (vnodeIsRoleLeader(pTq->pVnode)) { + if (msgType == TDMT_VND_SUBMIT) { + if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0; - void* data = taosMemoryMalloc(msgLen); - if (data == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("failed to copy data for stream since out of memory"); - return -1; + void* data = taosMemoryMalloc(msgLen); + if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("failed to copy data for stream since out of memory"); + return -1; + } + memcpy(data, msg, msgLen); + SSubmitReq* pReq = (SSubmitReq*)data; + pReq->version = ver; + + tqProcessSubmitReq(pTq, data, ver); + } + if (msgType == TDMT_VND_DELETE) { + tqProcessDelReq(pTq, POINTER_SHIFT(msg, sizeof(SMsgHead)), msgLen - sizeof(SMsgHead), ver); } - memcpy(data, msg, msgLen); - SSubmitReq* pReq = (SSubmitReq*)data; - pReq->version = ver; - - tqProcessStreamTrigger(pTq, data, ver); } return 0; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index cd0c7ba1f3..96e8e44bc9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -371,7 +371,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { if (NULL == pMetaRsp) { return; } - + strcpy(pMetaRsp->dbFName, pVnode->config.dbname); pMetaRsp->dbId = pVnode->config.dbId; pMetaRsp->vgId = TD_VID(pVnode); @@ -527,7 +527,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pR cRsp.code = TSDB_CODE_SUCCESS; tdFetchTbUidList(pVnode->pSma, &pStore, pCreateReq->ctb.suid, pCreateReq->uid); taosArrayPush(tbUids, &pCreateReq->uid); - vnodeUpdateMetaRsp(pVnode, cRsp.pMeta); + vnodeUpdateMetaRsp(pVnode, cRsp.pMeta); } taosArrayPush(rsp.pArray, &cRsp); @@ -1107,6 +1107,8 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq tDecoderInit(pCoder, pReq, len); tDecodeDeleteRes(pCoder, pRes); + ASSERT(pRes->skey != 0); + ASSERT(pRes->ekey != 0); for (int32_t iUid = 0; iUid < taosArrayGetSize(pRes->uidList); iUid++) { code = tsdbDeleteTableData(pVnode->pTsdb, version, pRes->suid, *(uint64_t *)taosArrayGet(pRes->uidList, iUid), diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 3bed81f657..c248015604 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -38,11 +38,11 @@ extern "C" { #include "tlockfree.h" #include "tmsg.h" #include "tpagedbuf.h" -#include "tstreamUpdate.h" #include "tstream.h" +#include "tstreamUpdate.h" -#include "vnode.h" #include "executorInt.h" +#include "vnode.h" typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); @@ -139,23 +139,23 @@ enum { }; typedef struct { - //TODO remove prepareStatus - STqOffsetVal prepareStatus; // for tmq - STqOffsetVal lastStatus; // for tmq - SMqMetaRsp metaRsp; // for tmq fetching meta - int8_t returned; - int64_t snapshotVer; + // TODO remove prepareStatus + STqOffsetVal prepareStatus; // for tmq + STqOffsetVal lastStatus; // for tmq + SMqMetaRsp metaRsp; // for tmq fetching meta + int8_t returned; + int64_t snapshotVer; - SSchemaWrapper *schema; - char tbName[TSDB_TABLE_NAME_LEN]; - SSDataBlock* pullOverBlk; // for streaming - SWalFilterCond cond; - int64_t lastScanUid; - int8_t recoverStep; + SSchemaWrapper* schema; + char tbName[TSDB_TABLE_NAME_LEN]; + SSDataBlock* pullOverBlk; // for streaming + SWalFilterCond cond; + int64_t lastScanUid; + int8_t recoverStep; SQueryTableDataCond tableCond; - int64_t recoverStartVer; - int64_t recoverEndVer; - SStreamState* pState; + int64_t recoverStartVer; + int64_t recoverEndVer; + SStreamState* pState; } SStreamTaskInfo; typedef struct { @@ -167,29 +167,29 @@ typedef struct { } SSchemaInfo; typedef struct SExecTaskInfo { - STaskIdInfo id; - uint32_t status; - STimeWindow window; - STaskCostInfo cost; - int64_t owner; // if it is in execution - int32_t code; + STaskIdInfo id; + uint32_t status; + STimeWindow window; + STaskCostInfo cost; + int64_t owner; // if it is in execution + int32_t code; - int64_t version; // used for stream to record wal version - SStreamTaskInfo streamInfo; - SSchemaInfo schemaInfo; - STableListInfo tableqinfoList; // this is a table list - const char* sql; // query sql string - jmp_buf env; // jump to this position when error happens. - EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] - SSubplan* pSubplan; + int64_t version; // used for stream to record wal version + SStreamTaskInfo streamInfo; + SSchemaInfo schemaInfo; + STableListInfo tableqinfoList; // this is a table list + const char* sql; // query sql string + jmp_buf env; // jump to this position when error happens. + EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] + SSubplan* pSubplan; struct SOperatorInfo* pRoot; } SExecTaskInfo; enum { - OP_NOT_OPENED = 0x0, - OP_OPENED = 0x1, + OP_NOT_OPENED = 0x0, + OP_OPENED = 0x1, OP_RES_TO_RETURN = 0x5, - OP_EXEC_DONE = 0x9, + OP_EXEC_DONE = 0x9, }; typedef struct SOperatorFpSet { @@ -228,7 +228,7 @@ typedef struct SOperatorInfo { typedef enum { EX_SOURCE_DATA_NOT_READY = 0x1, - EX_SOURCE_DATA_READY = 0x2, + EX_SOURCE_DATA_READY = 0x2, EX_SOURCE_DATA_EXHAUSTED = 0x3, } EX_SOURCE_STATUS; @@ -251,26 +251,26 @@ typedef struct SLoadRemoteDataInfo { } SLoadRemoteDataInfo; typedef struct SLimitInfo { - SLimit limit; - SLimit slimit; - uint64_t currentGroupId; - int64_t remainGroupOffset; - int64_t numOfOutputGroups; - int64_t remainOffset; - int64_t numOfOutputRows; + SLimit limit; + SLimit slimit; + uint64_t currentGroupId; + int64_t remainGroupOffset; + int64_t numOfOutputGroups; + int64_t remainOffset; + int64_t numOfOutputRows; } SLimitInfo; typedef struct SExchangeInfo { - SArray* pSources; - SArray* pSourceDataInfo; - tsem_t ready; - void* pTransporter; + SArray* pSources; + SArray* pSourceDataInfo; + tsem_t ready; + void* pTransporter; // SArray, result block list, used to keep the multi-block that // passed by downstream operator SArray* pResultBlockList; - int32_t rspBlockIndex; // indicate the return block index in pResultBlockList - SSDataBlock* pDummyBlock; // dummy block, not keep data - bool seqLoadData; // sequential load data or not, false by default + int32_t rspBlockIndex; // indicate the return block index in pResultBlockList + SSDataBlock* pDummyBlock; // dummy block, not keep data + bool seqLoadData; // sequential load data or not, false by default int32_t current; SLoadRemoteDataInfo loadInfo; uint64_t self; @@ -278,22 +278,22 @@ typedef struct SExchangeInfo { } SExchangeInfo; typedef struct SColMatchInfo { - int32_t srcSlotId; // source slot id + int32_t srcSlotId; // source slot id int32_t colId; int32_t targetSlotId; - bool output; // todo remove this? + bool output; // todo remove this? bool reserved; - int32_t matchType; // determinate the source according to col id or slot id + int32_t matchType; // determinate the source according to col id or slot id } SColMatchInfo; typedef struct SScanInfo { - int32_t numOfAsc; - int32_t numOfDesc; + int32_t numOfAsc; + int32_t numOfDesc; } SScanInfo; typedef struct SSampleExecInfo { - double sampleRatio; // data block sample ratio, 1 by default - uint32_t seed; // random seed value + double sampleRatio; // data block sample ratio, 1 by default + uint32_t seed; // random seed value } SSampleExecInfo; enum { @@ -305,39 +305,40 @@ typedef struct SAggSupporter { SSHashObj* pResultRowHashTable; // quick locate the window object for each result char* keyBuf; // window key buffer SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file - int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row - int32_t currentPageId; // current write page id + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row + int32_t currentPageId; // current write page id } SAggSupporter; typedef struct { - // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded. - SInterval interval; - SAggSupporter *pAggSup; - SExprSupp *pExprSup; // expr supporter of aggregate operator + // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if + // current data block needs to be loaded. + SInterval interval; + SAggSupporter* pAggSup; + SExprSupp* pExprSup; // expr supporter of aggregate operator } SAggOptrPushDownInfo; typedef struct STableScanInfo { - STsdbReader* dataReader; - SReadHandle readHandle; + STsdbReader* dataReader; + SReadHandle readHandle; SFileBlockLoadRecorder readRecorder; - SScanInfo scanInfo; - int32_t scanTimes; - SNode* pFilterNode; // filter info, which is push down by optimizer + SScanInfo scanInfo; + int32_t scanTimes; + SNode* pFilterNode; // filter info, which is push down by optimizer - SSDataBlock* pResBlock; - SArray* pColMatchInfo; - SExprSupp pseudoSup; - SQueryTableDataCond cond; - int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan - int32_t dataBlockLoadFlag; - SSampleExecInfo sample; // sample execution info - int32_t currentGroupId; - int32_t currentTable; - int8_t scanMode; - int8_t noTable; + SSDataBlock* pResBlock; + SArray* pColMatchInfo; + SExprSupp pseudoSup; + SQueryTableDataCond cond; + int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan + int32_t dataBlockLoadFlag; + SSampleExecInfo sample; // sample execution info + int32_t currentGroupId; + int32_t currentTable; + int8_t scanMode; + int8_t noTable; SAggOptrPushDownInfo pdInfo; - int8_t assignBlockUid; + int8_t assignBlockUid; } STableScanInfo; typedef struct STableMergeScanInfo { @@ -370,7 +371,7 @@ typedef struct STableMergeScanInfo { SArray* pColMatchInfo; int32_t numOfOutput; - SExprSupp pseudoSup; + SExprSupp pseudoSup; SQueryTableDataCond cond; int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan @@ -384,26 +385,26 @@ typedef struct STableMergeScanInfo { } STableMergeScanInfo; typedef struct STagScanInfo { - SColumnInfo *pCols; - SSDataBlock *pRes; - SArray *pColMatchInfo; - int32_t curPos; - SReadHandle readHandle; - STableListInfo *pTableList; + SColumnInfo* pCols; + SSDataBlock* pRes; + SArray* pColMatchInfo; + int32_t curPos; + SReadHandle readHandle; + STableListInfo* pTableList; } STagScanInfo; typedef struct SLastrowScanInfo { - SSDataBlock *pRes; - SReadHandle readHandle; - void *pLastrowReader; - SArray *pColMatchInfo; - int32_t *pSlotIds; - SExprSupp pseudoExprSup; - int32_t retrieveType; - int32_t currentGroupIndex; - SSDataBlock *pBufferredRes; - SArray *pUidList; - int32_t indexOfBufferedRes; + SSDataBlock* pRes; + SReadHandle readHandle; + void* pLastrowReader; + SArray* pColMatchInfo; + int32_t* pSlotIds; + SExprSupp pseudoExprSup; + int32_t retrieveType; + int32_t currentGroupIndex; + SSDataBlock* pBufferredRes; + SArray* pUidList; + int32_t indexOfBufferedRes; } SLastrowScanInfo; typedef enum EStreamScanMode { @@ -432,10 +433,10 @@ typedef struct SStreamAggSupporter { SArray* pCurWins; int32_t valueSize; int32_t keySize; - char* pKeyBuf; // window key buffer - SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file - int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row - int32_t currentPageId; // buffer page that is active + char* pKeyBuf; // window key buffer + SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row + int32_t currentPageId; // buffer page that is active SSDataBlock* pScanBlock; } SStreamAggSupporter; @@ -447,22 +448,22 @@ typedef struct SWindowSupporter { } SWindowSupporter; typedef struct SPartitionBySupporter { - SArray* pGroupCols; // group by columns, SArray - SArray* pGroupColVals; // current group column values, SArray - char* keyBuf; // group by keys for hash - bool needCalc; // partition by column + SArray* pGroupCols; // group by columns, SArray + SArray* pGroupColVals; // current group column values, SArray + char* keyBuf; // group by keys for hash + bool needCalc; // partition by column } SPartitionBySupporter; typedef struct SPartitionDataInfo { - uint64_t groupId; - SArray* rowIds; + uint64_t groupId; + SArray* rowIds; } SPartitionDataInfo; typedef struct STimeWindowSupp { - int8_t calTrigger; - int64_t waterMark; - TSKEY maxTs; - SColumnInfoData timeWindowData; // query time window info for scalar function execution. + int8_t calTrigger; + int64_t waterMark; + TSKEY maxTs; + SColumnInfoData timeWindowData; // query time window info for scalar function execution. } STimeWindowAggSupp; typedef struct SStreamScanInfo { @@ -487,22 +488,22 @@ typedef struct SStreamScanInfo { uint64_t groupId; SUpdateInfo* pUpdateInfo; - EStreamScanMode scanMode; - SOperatorInfo* pStreamScanOp; - SOperatorInfo* pTableScanOp; - SArray* childIds; - SWindowSupporter windowSup; - SPartitionBySupporter partitionSup; - SExprSupp* pPartScalarSup; - bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA. - int32_t scanWinIndex; // for state operator - int32_t pullDataResIndex; - SSDataBlock* pPullDataRes; // pull data SSDataBlock - SSDataBlock* pDeleteDataRes; // delete data SSDataBlock - int32_t deleteDataIndex; - STimeWindow updateWin; - STimeWindowAggSupp twAggSup; - SSDataBlock* pUpdateDataRes; + EStreamScanMode scanMode; + SOperatorInfo* pStreamScanOp; + SOperatorInfo* pTableScanOp; + SArray* childIds; + SWindowSupporter windowSup; + SPartitionBySupporter partitionSup; + SExprSupp* pPartScalarSup; + bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA. + int32_t scanWinIndex; // for state operator + int32_t pullDataResIndex; + SSDataBlock* pPullDataRes; // pull data SSDataBlock + SSDataBlock* pDeleteDataRes; // delete data SSDataBlock + int32_t deleteDataIndex; + STimeWindow updateWin; + STimeWindowAggSupp twAggSup; + SSDataBlock* pUpdateDataRes; // status for tmq SNodeList* pGroupTags; SNode* pTagCond; @@ -571,17 +572,17 @@ typedef struct SIntervalAggOperatorInfo { EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] STimeWindowAggSupp twAggSup; bool invertible; - SArray* pPrevValues; // SArray used to keep the previous not null value for interpolation. + SArray* pPrevValues; // SArray used to keep the previous not null value for interpolation. bool ignoreExpiredData; SArray* pRecycledPages; - SArray* pDelWins; // SWinRes + SArray* pDelWins; // SWinRes int32_t delIndex; SSDataBlock* pDelRes; SNode* pCondition; } SIntervalAggOperatorInfo; typedef struct SMergeAlignedIntervalAggOperatorInfo { - SIntervalAggOperatorInfo *intervalAggOperatorInfo; + SIntervalAggOperatorInfo* intervalAggOperatorInfo; bool hasGroupId; uint64_t groupId; // current groupId @@ -592,63 +593,63 @@ typedef struct SMergeAlignedIntervalAggOperatorInfo { typedef struct SStreamFinalIntervalOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; // basic info - SAggSupporter aggSup; // aggregate supporter - SExprSupp scalarSupp; // supporter for perform scalar function - SGroupResInfo groupResInfo; // multiple results build supporter - SInterval interval; // interval info - int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. - int32_t order; // current SSDataBlock scan order + SOptrBasicInfo binfo; // basic info + SAggSupporter aggSup; // aggregate supporter + SExprSupp scalarSupp; // supporter for perform scalar function + SGroupResInfo groupResInfo; // multiple results build supporter + SInterval interval; // interval info + int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. + int32_t order; // current SSDataBlock scan order STimeWindowAggSupp twAggSup; SArray* pChildren; SSDataBlock* pUpdateRes; bool returnUpdate; - SPhysiNode* pPhyNode; // create new child + SPhysiNode* pPhyNode; // create new child bool isFinal; SHashObj* pPullDataMap; - SArray* pPullWins; // SPullWindowInfo + SArray* pPullWins; // SPullWindowInfo int32_t pullIndex; SSDataBlock* pPullDataRes; bool ignoreExpiredData; SArray* pRecycledPages; - SArray* pDelWins; // SWinRes + SArray* pDelWins; // SWinRes int32_t delIndex; SSDataBlock* pDelRes; } SStreamFinalIntervalOperatorInfo; typedef struct SAggOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; + SOptrBasicInfo binfo; + SAggSupporter aggSup; - STableQueryInfo *current; - uint64_t groupId; - SGroupResInfo groupResInfo; - SExprSupp scalarExprSup; - SNode *pCondition; + STableQueryInfo* current; + uint64_t groupId; + SGroupResInfo groupResInfo; + SExprSupp scalarExprSup; + SNode* pCondition; } SAggOperatorInfo; typedef struct SProjectOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SNode* pFilterNode; // filter info, which is push down by optimizer - SArray* pPseudoColInfo; - SLimitInfo limitInfo; - bool mergeDataBlocks; - SSDataBlock* pFinalRes; - SNode* pCondition; + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SNode* pFilterNode; // filter info, which is push down by optimizer + SArray* pPseudoColInfo; + SLimitInfo limitInfo; + bool mergeDataBlocks; + SSDataBlock* pFinalRes; + SNode* pCondition; } SProjectOperatorInfo; typedef struct SIndefOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SArray* pPseudoColInfo; - SExprSupp scalarSup; - SNode* pCondition; - uint64_t groupId; + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SArray* pPseudoColInfo; + SExprSupp scalarSup; + SNode* pCondition; + uint64_t groupId; - SSDataBlock* pNextGroupRes; + SSDataBlock* pNextGroupRes; } SIndefOperatorInfo; typedef struct SFillOperatorInfo { @@ -663,7 +664,7 @@ typedef struct SFillOperatorInfo { SArray* pColMatchColInfo; int32_t primaryTsCol; int32_t primarySrcSlotId; - uint64_t curGroupId; // current handled group id + uint64_t curGroupId; // current handled group id SExprInfo* pExprInfo; int32_t numOfExpr; SExprInfo* pNotFillExprInfo; @@ -672,23 +673,23 @@ typedef struct SFillOperatorInfo { typedef struct SGroupbyOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; + SOptrBasicInfo binfo; + SAggSupporter aggSup; - SArray* pGroupCols; // group by columns, SArray - SArray* pGroupColVals; // current group column values, SArray - SNode* pCondition; - bool isInit; // denote if current val is initialized or not - char* keyBuf; // group by keys for hash - int32_t groupKeyLen; // total group by column width - SGroupResInfo groupResInfo; - SExprSupp scalarSup; + SArray* pGroupCols; // group by columns, SArray + SArray* pGroupColVals; // current group column values, SArray + SNode* pCondition; + bool isInit; // denote if current val is initialized or not + char* keyBuf; // group by keys for hash + int32_t groupKeyLen; // total group by column width + SGroupResInfo groupResInfo; + SExprSupp scalarSup; } SGroupbyOperatorInfo; typedef struct SDataGroupInfo { - uint64_t groupId; - int64_t numOfRows; - SArray* pPageList; + uint64_t groupId; + int64_t numOfRows; + SArray* pPageList; } SDataGroupInfo; // The sort in partition may be needed later. @@ -700,12 +701,12 @@ typedef struct SPartitionOperatorInfo { int32_t groupKeyLen; // total group by column width SHashObj* pGroupSet; // quick locate the window object for each result - SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file - int32_t rowCapacity; // maximum number of rows for each buffer page - int32_t* columnOffset; // start position for each column data - SArray* sortedGroupArray; // SDataGroupInfo sorted by group id - int32_t groupIndex; // group index - int32_t pageIndex; // page index of current group + SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file + int32_t rowCapacity; // maximum number of rows for each buffer page + int32_t* columnOffset; // start position for each column data + SArray* sortedGroupArray; // SDataGroupInfo sorted by group id + int32_t groupIndex; // group index + int32_t pageIndex; // page index of current group SExprSupp scalarSup; } SPartitionOperatorInfo; @@ -719,52 +720,52 @@ typedef struct SWindowRowsSup { typedef struct SSessionAggOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; + SOptrBasicInfo binfo; + SAggSupporter aggSup; SGroupResInfo groupResInfo; SWindowRowsSup winSup; - bool reptScan; // next round scan - int64_t gap; // session window gap - int32_t tsSlotId; // primary timestamp slot id + bool reptScan; // next round scan + int64_t gap; // session window gap + int32_t tsSlotId; // primary timestamp slot id STimeWindowAggSupp twAggSup; - const SNode* pCondition; + const SNode* pCondition; } SSessionAggOperatorInfo; typedef struct SResultWindowInfo { SResultRowPosition pos; - STimeWindow win; - uint64_t groupId; - bool isOutput; - bool isClosed; + STimeWindow win; + uint64_t groupId; + bool isOutput; + bool isClosed; } SResultWindowInfo; typedef struct SStateWindowInfo { SResultWindowInfo winInfo; - SStateKeys stateKey; + SStateKeys stateKey; } SStateWindowInfo; typedef struct SStreamSessionAggOperatorInfo { - SOptrBasicInfo binfo; - SStreamAggSupporter streamAggSup; - SExprSupp scalarSupp; // supporter for perform scalar function - SGroupResInfo groupResInfo; - int64_t gap; // session window gap - int32_t primaryTsIndex; // primary timestamp slot id - int32_t endTsIndex; // window end timestamp slot id - int32_t order; // current SSDataBlock scan order - STimeWindowAggSupp twAggSup; - SSDataBlock* pWinBlock; // window result - SqlFunctionCtx* pDummyCtx; // for combine - SSDataBlock* pDelRes; // delete result - SSDataBlock* pUpdateRes; // update window - bool returnUpdate; - SHashObj* pStDeleted; - void* pDelIterator; - SArray* pChildren; // cache for children's result; final stream operator - SPhysiNode* pPhyNode; // create new child - bool isFinal; - bool ignoreExpiredData; + SOptrBasicInfo binfo; + SStreamAggSupporter streamAggSup; + SExprSupp scalarSupp; // supporter for perform scalar function + SGroupResInfo groupResInfo; + int64_t gap; // session window gap + int32_t primaryTsIndex; // primary timestamp slot id + int32_t endTsIndex; // window end timestamp slot id + int32_t order; // current SSDataBlock scan order + STimeWindowAggSupp twAggSup; + SSDataBlock* pWinBlock; // window result + SqlFunctionCtx* pDummyCtx; // for combine + SSDataBlock* pDelRes; // delete result + SSDataBlock* pUpdateRes; // update window + bool returnUpdate; + SHashObj* pStDeleted; + void* pDelIterator; + SArray* pChildren; // cache for children's result; final stream operator + SPhysiNode* pPhyNode; // create new child + bool isFinal; + bool ignoreExpiredData; } SStreamSessionAggOperatorInfo; typedef struct SStreamPartitionOperatorInfo { @@ -778,26 +779,26 @@ typedef struct SStreamPartitionOperatorInfo { } SStreamPartitionOperatorInfo; typedef struct STimeSliceOperatorInfo { - SSDataBlock* pRes; - STimeWindow win; - SInterval interval; - int64_t current; - SArray* pPrevRow; // SArray - SArray* pNextRow; // SArray - SArray* pLinearInfo; // SArray - bool fillLastPoint; - bool isPrevRowSet; - bool isNextRowSet; - int32_t fillType; // fill type - SColumn tsCol; // primary timestamp column - SExprSupp scalarSup; // scalar calculation - struct SFillColInfo* pFillColInfo; // fill column info + SSDataBlock* pRes; + STimeWindow win; + SInterval interval; + int64_t current; + SArray* pPrevRow; // SArray + SArray* pNextRow; // SArray + SArray* pLinearInfo; // SArray + bool fillLastPoint; + bool isPrevRowSet; + bool isNextRowSet; + int32_t fillType; // fill type + SColumn tsCol; // primary timestamp column + SExprSupp scalarSup; // scalar calculation + struct SFillColInfo* pFillColInfo; // fill column info } STimeSliceOperatorInfo; typedef struct SStateWindowOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; + SOptrBasicInfo binfo; + SAggSupporter aggSup; SGroupResInfo groupResInfo; SWindowRowsSup winSup; @@ -811,52 +812,52 @@ typedef struct SStateWindowOperatorInfo { } SStateWindowOperatorInfo; typedef struct SStreamStateAggOperatorInfo { - SOptrBasicInfo binfo; - SStreamAggSupporter streamAggSup; - SExprSupp scalarSupp; // supporter for perform scalar function - SGroupResInfo groupResInfo; - int32_t primaryTsIndex; // primary timestamp slot id - int32_t order; // current SSDataBlock scan order - STimeWindowAggSupp twAggSup; - SColumn stateCol; - SqlFunctionCtx* pDummyCtx; // for combine - SSDataBlock* pDelRes; - SHashObj* pSeDeleted; - void* pDelIterator; - SArray* pChildren; // cache for children's result; - bool ignoreExpiredData; + SOptrBasicInfo binfo; + SStreamAggSupporter streamAggSup; + SExprSupp scalarSupp; // supporter for perform scalar function + SGroupResInfo groupResInfo; + int32_t primaryTsIndex; // primary timestamp slot id + int32_t order; // current SSDataBlock scan order + STimeWindowAggSupp twAggSup; + SColumn stateCol; + SqlFunctionCtx* pDummyCtx; // for combine + SSDataBlock* pDelRes; + SHashObj* pSeDeleted; + void* pDelIterator; + SArray* pChildren; // cache for children's result; + bool ignoreExpiredData; } SStreamStateAggOperatorInfo; typedef struct SSortedMergeOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; + SOptrBasicInfo binfo; + SAggSupporter aggSup; - SArray* pSortInfo; - int32_t numOfSources; - SSortHandle *pSortHandle; - int32_t bufPageSize; - uint32_t sortBufSize; // max buffer size for in-memory sort - int32_t resultRowFactor; - bool hasGroupVal; - SDiskbasedBuf *pTupleStore; // keep the final results - int32_t numOfResPerPage; - char** groupVal; - SArray *groupInfo; + SArray* pSortInfo; + int32_t numOfSources; + SSortHandle* pSortHandle; + int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort + int32_t resultRowFactor; + bool hasGroupVal; + SDiskbasedBuf* pTupleStore; // keep the final results + int32_t numOfResPerPage; + char** groupVal; + SArray* groupInfo; } SSortedMergeOperatorInfo; typedef struct SSortOperatorInfo { SOptrBasicInfo binfo; - uint32_t sortBufSize; // max buffer size for in-memory sort - SArray* pSortInfo; - SSortHandle* pSortHandle; - SArray* pColMatchInfo; // for index map from table scan output - int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort + SArray* pSortInfo; + SSortHandle* pSortHandle; + SArray* pColMatchInfo; // for index map from table scan output + int32_t bufPageSize; - int64_t startTs; // sort start time - uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. - SLimitInfo limitInfo; - SNode* pCondition; + int64_t startTs; // sort start time + uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. + SLimitInfo limitInfo; + SNode* pCondition; } SSortOperatorInfo; typedef struct STagFilterOperatorInfo { @@ -864,18 +865,18 @@ typedef struct STagFilterOperatorInfo { } STagFilterOperatorInfo; typedef struct SJoinOperatorInfo { - SSDataBlock *pRes; - int32_t joinType; - int32_t inputOrder; + SSDataBlock* pRes; + int32_t joinType; + int32_t inputOrder; - SSDataBlock *pLeft; - int32_t leftPos; - SColumnInfo leftCol; + SSDataBlock* pLeft; + int32_t leftPos; + SColumnInfo leftCol; - SSDataBlock *pRight; - int32_t rightPos; - SColumnInfo rightCol; - SNode *pCondAfterMerge; + SSDataBlock* pRight; + int32_t rightPos; + SColumnInfo rightCol; + SNode* pCondAfterMerge; } SJoinOperatorInfo; #define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) @@ -884,8 +885,8 @@ typedef struct SJoinOperatorInfo { void doDestroyExchangeOperatorInfo(void* param); SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, - __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode, - __optr_decode_fn_t decode, __optr_explain_fn_t explain); + __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode, + __optr_decode_fn_t decode, __optr_explain_fn_t explain); int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); void operatorDummyCloseFn(void* param, int32_t numOfCols); @@ -896,24 +897,26 @@ void cleanupBasicInfo(SOptrBasicInfo* pInfo); int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr); void cleanupExprSupp(SExprSupp* pSup); void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs); -int32_t initAggInfo(SExprSupp *pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, +int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, const char* pkey); -void initResultSizeInfo(SResultInfo * pResultInfo, int32_t numOfRows); -void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf); -int32_t handleLimitOffset(SOperatorInfo *pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf); +void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows); +void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, + SDiskbasedBuf* pBuf); +int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf); bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo); void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo); -void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, int32_t offset, - int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput); +void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, int32_t offset, + int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput); -int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, int32_t numOfOutput, SArray* pColList, char** pNextStart); -void updateLoadRemoteInfo(SLoadRemoteDataInfo *pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, - SOperatorInfo* pOperator); +int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, int32_t numOfOutput, SArray* pColList, + char** pNextStart); +void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, + SOperatorInfo* pOperator); STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); -int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag); +int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag); int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); void doSetOperatorCompleted(SOperatorInfo* pOperator); @@ -921,10 +924,10 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pC int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, SSDataBlock* pBlock, const char* idStr); -void cleanupAggSup(SAggSupporter* pAggSup); -void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); -void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); -void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId); +void cleanupAggSup(SAggSupporter* pAggSup); +void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); +void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); +void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId); int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts); int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts); @@ -933,29 +936,37 @@ SSDataBlock* loadNextDataBlock(void* param); void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset); -SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, - char* pData, int16_t bytes, bool masterscan, uint64_t groupId, - SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup); +SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, + int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, + bool isIntervalQuery, SAggSupporter* pSup); SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, + SExecTaskInfo* pTaskInfo); SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, + const char* pUser, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo, +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, bool mergeResult, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, + SExecTaskInfo* pTaskInfo); +SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, + SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, + SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, + SExecTaskInfo* pTaskInfo); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream); + STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo, bool isStream); SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMergeAlignedIntervalPhysiNode* pNode, @@ -975,33 +986,39 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, +SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, + SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode, + SExecTaskInfo* pTaskInfo); +SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, - SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, - SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); +SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, + SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, + SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo, int32_t numOfChild); + +SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo); int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, - int32_t numOfOutput, SArray* pPseudoList); + int32_t numOfOutput, SArray* pPseudoList); -void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol); +void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, + int32_t scanFlag, bool createDummyCol); bool isTaskKilled(SExecTaskInfo* pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); -void setTaskKilled(SExecTaskInfo* pTaskInfo); -void queryCostStatis(SExecTaskInfo* pTaskInfo); +void setTaskKilled(SExecTaskInfo* pTaskInfo); +void queryCostStatis(SExecTaskInfo* pTaskInfo); void doDestroyTask(SExecTaskInfo* pTaskInfo); int32_t getMaximumIdleDurationSec(); @@ -1013,7 +1030,7 @@ int32_t getMaximumIdleDurationSec(); * nOptrWithVal: *nOptrWithVal save the number of optr with value * return: result code, 0 means success */ -int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length, int32_t *nOptrWithVal); +int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t* length, int32_t* nOptrWithVal); /* * ops: root operator, created by caller @@ -1026,7 +1043,7 @@ int32_t decodeOperator(SOperatorInfo* ops, const char* data, int32_t length); void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, char* sql, EOPTR_EXEC_MODEL model); -int32_t createDataSinkParam(SDataSinkNode *pNode, void **pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle); +int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList); int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result); @@ -1035,42 +1052,43 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, int32_t order); int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, - __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); + __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput, - int32_t size); -SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize); -SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, - TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex); -SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, - TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex); -bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); -bool functionNeedToExecute(SqlFunctionCtx* pCtx); -bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); -bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); -bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup); -void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp); -void printDataBlock(SSDataBlock* pBlock, const char* flag); + int32_t size); +SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize); +SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, + int64_t gap, int32_t* pIndex); +SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, + int64_t gap, int32_t* pIndex); +bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); +bool functionNeedToExecute(SqlFunctionCtx* pCtx); +bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); +bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); +bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup); +void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp); +void printDataBlock(SSDataBlock* pBlock, const char* flag); uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, - SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, - SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); + SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, + const int32_t* rowCellOffset, SSDataBlock* pBlock, + SExecTaskInfo* pTaskInfo); -int32_t createScanTableListInfo(SScanPhysiNode *pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, - STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* idstr); +int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, + STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, + const char* idstr); SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo, +SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); -bool groupbyTbname(SNodeList* pGroupList); +bool groupbyTbname(SNodeList* pGroupList); int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); -SSDataBlock* createSpecialDataBlock(EStreamType type); -void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput); +void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput); #ifdef __cplusplus } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b740ec21d3..60670b1a49 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -955,9 +955,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) { pTableScanInfo->currentGroupId = -1; } -static void freeArray(void* array) { - taosArrayDestroy(array); -} +static void freeArray(void* array) { taosArrayDestroy(array); } static void resetTableScanOperator(SOperatorInfo* pTableScanOp) { STableScanInfo* pTableScanInfo = pTableScanOp->info; @@ -971,15 +969,16 @@ static void resetTableScanOperator(SOperatorInfo* pTableScanOp) { resetTableScanInfo(pTableScanOp->info, &win); } -static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs, int64_t maxVersion) { +static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs, + int64_t maxVersion) { SArray* gpTbls = pTableScanOp->pTaskInfo->tableqinfoList.pGroupList; taosArrayClear(gpTbls); STableKeyInfo tblInfo = {.uid = tbUid, .groupId = 0}; - SArray* tbls = taosArrayInit(1, sizeof(STableKeyInfo)); + SArray* tbls = taosArrayInit(1, sizeof(STableKeyInfo)); taosArrayPush(tbls, &tblInfo); taosArrayPush(gpTbls, &tbls); - STimeWindow win = {.skey = startTs, .ekey = endTs}; + STimeWindow win = {.skey = startTs, .ekey = endTs}; STableScanInfo* pTableScanInfo = pTableScanOp->info; pTableScanInfo->cond.startVersion = -1; pTableScanInfo->cond.endVersion = maxVersion; @@ -1068,8 +1067,8 @@ static STimeWindow getSlidingWindow(TSKEY* tsCol, SInterval* pInterval, SDataBlo if (hasGroup) { (*pRowIndex) += 1; } else { - (*pRowIndex) += getNumOfRowsInTimeWindow(pDataBlockInfo, tsCol, *pRowIndex, endWin.ekey, - binarySearchForKey, NULL, TSDB_ORDER_ASC); + (*pRowIndex) += getNumOfRowsInTimeWindow(pDataBlockInfo, tsCol, *pRowIndex, endWin.ekey, binarySearchForKey, NULL, + TSDB_ORDER_ASC); } do { preWin = endWin; @@ -1110,8 +1109,8 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 for (int32_t j = 0; j < pInfo->pTableScanOp->exprSupp.numOfExprs; j++) { SColumnInfoData* pSrcCol = taosArrayGet(tmpBlock->pDataBlock, j); SColumnInfoData* pDestCol = taosArrayGet(pResult->pDataBlock, j); - bool isNull = colDataIsNull(pSrcCol, tmpBlock->info.rows, i, NULL); - char* pSrcData = colDataGetData(pSrcCol, i); + bool isNull = colDataIsNull(pSrcCol, tmpBlock->info.rows, i, NULL); + char* pSrcData = colDataGetData(pSrcCol, i); colDataAppend(pDestCol, pResult->info.rows, pSrcData, isNull); } pResult->info.rows++; @@ -1152,7 +1151,7 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); int32_t dummy = 0; - int64_t version = pSrcBlock->info.version - 1; + int64_t version = pSrcBlock->info.version - 1; for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], version); // gap must be 0. @@ -1201,15 +1200,15 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS SColumnInfoData* pGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX); SColumnInfoData* pCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); - int64_t version = pSrcBlock->info.version - 1; + int64_t version = pSrcBlock->info.version - 1; for (int32_t i = 0; i < rows;) { uint64_t srcUid = srcUidData[i]; uint64_t groupId = getGroupIdByData(pInfo, srcUid, tsCol[i], version); uint64_t srcGpId = srcGp[i]; - TSKEY calStartTs = tsCol[i]; + TSKEY calStartTs = tsCol[i]; colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false); STimeWindow win = getSlidingWindow(tsCol, &pInfo->interval, &pSrcBlock->info, &i, pInfo->partitionSup.needCalc); - TSKEY calEndTs = tsCol[i - 1]; + TSKEY calEndTs = tsCol[i - 1]; colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false); colDataAppend(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false); colDataAppend(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false); @@ -1277,10 +1276,10 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock bool closedWin = isClosed && isSignleIntervalWindow(pInfo) && isDeletedWindow(&win, pBlock->info.groupId, pInfo->windowSup.pIntervalAggSup); if ((update || closedWin) && out) { - uint64_t gpId = closedWin&&pInfo->partitionSup.needCalc ? - calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId) : 0; - appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid, - &gpId); + uint64_t gpId = closedWin && pInfo->partitionSup.needCalc + ? calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId) + : 0; + appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid, &gpId); } } if (out && pInfo->pUpdateDataRes->info.rows > 0) { @@ -1515,6 +1514,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { generateScanRange(pInfo, pBlock, pInfo->pUpdateRes); prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); + pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA; pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; return pInfo->pDeleteDataRes; } break; @@ -1915,7 +1915,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys ASSERT(pHandle->tqReader); pInfo->tqReader = pHandle->tqReader; } - + pInfo->pUpdateInfo = NULL; pInfo->pTableScanOp = pTableScanOp; pInfo->interval = pTSInfo->pdInfo.interval; @@ -1947,8 +1947,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR); pInfo->pCondition = pScanPhyNode->node.pConditions; pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; - pInfo->windowSup = - (SWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN}; + pInfo->windowSup = (SWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN}; pInfo->groupId = 0; pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE); pInfo->pStreamScanOp = pOperator; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 26667cd64f..ab28c91867 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -35,14 +35,15 @@ typedef struct SPullWindowInfo { typedef struct SOpenWindowInfo { SResultRowPosition pos; - uint64_t groupId; + uint64_t groupId; } SOpenWindowInfo; static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator); static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo); -static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult, uint64_t groupId); +static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult, + uint64_t groupId); static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult); ///* @@ -601,15 +602,15 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; SExprSupp* pSup = &pOperatorInfo->exprSupp; - int32_t startPos = 0; - int32_t numOfOutput = pSup->numOfExprs; + int32_t startPos = 0; + int32_t numOfOutput = pSup->numOfExprs; SResultRow* pResult = NULL; while (1) { - SListNode* pn = tdListGetHead(pResultRowInfo->openWindow); - SOpenWindowInfo* pOpenWin = (SOpenWindowInfo *)pn->data; - uint64_t groupId = pOpenWin->groupId; + SListNode* pn = tdListGetHead(pResultRowInfo->openWindow); + SOpenWindowInfo* pOpenWin = (SOpenWindowInfo*)pn->data; + uint64_t groupId = pOpenWin->groupId; SResultRowPosition* p1 = &pOpenWin->pos; if (p->pageId == p1->pageId && p->offset == p1->offset) { break; @@ -1028,7 +1029,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder); // window start(end) key interpolation doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup); - //TODO: add to open window? how to close the open windows after input blocks exhausted? + // TODO: add to open window? how to close the open windows after input blocks exhausted? #if 0 if ((ascScan && ekey <= pBlock->info.window.ekey) || (!ascScan && ekey >= pBlock->info.window.skey)) { @@ -1062,13 +1063,13 @@ SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SRe openWin.pos.pageId = pResult->pageId; openWin.pos.offset = pResult->offset; openWin.groupId = groupId; - SListNode* pn = tdListGetTail(pResultRowInfo->openWindow); + SListNode* pn = tdListGetTail(pResultRowInfo->openWindow); if (pn == NULL) { tdListAppend(pResultRowInfo->openWindow, &openWin); return openWin.pos; } - SOpenWindowInfo * px = (SOpenWindowInfo *)pn->data; + SOpenWindowInfo* px = (SOpenWindowInfo*)pn->data; if (px->pos.pageId != openWin.pos.pageId || px->pos.offset != openWin.pos.offset || px->groupId != openWin.groupId) { tdListAppend(pResultRowInfo->openWindow, &openWin); } @@ -1424,7 +1425,8 @@ bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) return true; } -void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* pDelWins, SInterval* pInterval, SHashObj* pUpdatedMap) { +void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* pDelWins, SInterval* pInterval, + SHashObj* pUpdatedMap) { SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* tsStarts = (TSKEY*)pStartCol->pData; SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); @@ -1469,7 +1471,6 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* } static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) { - void* pIte = NULL; size_t keyLen = 0; int32_t iter = 0; @@ -1826,7 +1827,8 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt return needed; } -void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup, SInterval* pInterval, int64_t waterMark) { +void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup, SInterval* pInterval, + int64_t waterMark) { if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { initIntervalDownStream(downstream->pDownstream[0], type, pSup, pInterval, waterMark); return; @@ -1921,7 +1923,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL) { - initIntervalDownStream(downstream, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark); + initIntervalDownStream(downstream, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, &pInfo->aggSup, &pInfo->interval, + pInfo->twAggSup.waterMark); } code = appendDownstream(pOperator, &downstream, 1); @@ -2248,7 +2251,6 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp if (hasInterp) { pResBlock->info.rows += 1; } - } static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { @@ -2721,7 +2723,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi goto _error; } - int32_t num = 0; + int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createResDataBlock(pStateNode->window.node.pOutputDataBlockDesc); int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; @@ -2748,7 +2750,9 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi initBasicInfo(&pInfo->binfo, pResBlock); initResultRowInfo(&pInfo->binfo.resultRowInfo); - pInfo->twAggSup = (STimeWindowAggSupp){.waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType};; + pInfo->twAggSup = + (STimeWindowAggSupp){.waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType}; + ; initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); pInfo->tsSlotId = tsSlotId; @@ -2857,8 +2861,8 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 } SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pDestCtx[k]); - char* p = GET_ROWCELL_INTERBUF(pEntryInfo); - SColumnInfoData idata = {0}; + char* p = GET_ROWCELL_INTERBUF(pEntryInfo); + SColumnInfoData idata = {0}; idata.info.type = TSDB_DATA_TYPE_BIGINT; idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; idata.pData = p; @@ -2867,7 +2871,7 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData}; pDestCtx[k].sfp.process(&tw, 1, &out); pEntryInfo->numOfRes = 1; - }else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) { + } else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) { int32_t code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]); if (code != TSDB_CODE_SUCCESS) { qError("%s apply functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code)); @@ -2893,7 +2897,8 @@ STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) { } static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, SArray* pWinArray, - int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SHashObj* pUpdatedMap) { + int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, + SHashObj* pUpdatedMap) { int32_t size = taosArrayGetSize(pWinArray); if (!pInfo->pChildren) { return; @@ -2934,7 +2939,7 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) { SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId); SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf, - GET_RES_WINDOW_KEY_LEN(sizeof(int64_t))); + GET_RES_WINDOW_KEY_LEN(sizeof(int64_t))); return p1 == NULL; } @@ -3059,7 +3064,7 @@ static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) tSimpleHashClear(pInfo->aggSup.pResultRowHashTable); clearDiskbasedBuf(pInfo->aggSup.pResultBuf); initResultRowInfo(&pInfo->binfo.resultRowInfo); - pInfo->aggSup.currentPageId = -1; + pInfo->aggSup.currentPageId = -1; } static void clearSpecialDataBlock(SSDataBlock* pBlock) { @@ -3145,7 +3150,7 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) { static void addRetriveWindow(SArray* wins, SStreamFinalIntervalOperatorInfo* pInfo) { int32_t size = taosArrayGetSize(wins); for (int32_t i = 0; i < size; i++) { - SWinKey* winKey = taosArrayGet(wins, i); + SWinKey* winKey = taosArrayGet(wins, i); STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval); if (isCloseWindow(&nextWin, &pInfo->twAggSup) && !pInfo->ignoreExpiredData) { void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey)); @@ -3274,8 +3279,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info; SExprSupp* pChildSup = &pChildOp->exprSupp; doDeleteSpecifyIntervalWindow(&pChildInfo->aggSup, pBlock, NULL, &pChildInfo->interval, NULL); - rebuildIntervalWindow(pInfo, pSup, delWins, pInfo->binfo.pRes->info.groupId, - pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, pUpdatedMap); + rebuildIntervalWindow(pInfo, pSup, delWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs, + pOperator->pTaskInfo, pUpdatedMap); addRetriveWindow(delWins, pInfo); taosArrayAddAll(pInfo->pDelWins, delWins); taosArrayDestroy(delWins); @@ -3381,40 +3386,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return NULL; } -SSDataBlock* createSpecialDataBlock(EStreamType type) { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.hasVarCol = false; - pBlock->info.groupId = 0; - pBlock->info.rows = 0; - pBlock->info.type = type; - pBlock->info.rowSize = - sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t) + sizeof(uint64_t) + sizeof(TSKEY) + sizeof(TSKEY); - pBlock->info.watermark = INT64_MIN; - - pBlock->pDataBlock = taosArrayInit(6, sizeof(SColumnInfoData)); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP; - infoData.info.bytes = sizeof(TSKEY); - // window start ts - taosArrayPush(pBlock->pDataBlock, &infoData); - // window end ts - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_UBIGINT; - infoData.info.bytes = sizeof(uint64_t); - // uid - taosArrayPush(pBlock->pDataBlock, &infoData); - // group id - taosArrayPush(pBlock->pDataBlock, &infoData); - - // calculate start ts - taosArrayPush(pBlock->pDataBlock, &infoData); - // calculate end ts - taosArrayPush(pBlock->pDataBlock, &infoData); - - return pBlock; -} - SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; @@ -3709,7 +3680,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); if (downstream) { - initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType, pInfo->primaryTsIndex); + initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType, + pInfo->primaryTsIndex); code = appendDownstream(pOperator, &downstream, 1); } return pOperator; @@ -3740,12 +3712,14 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) { bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { return isInTimeWindow(&pWinInfo->win, ts, gap); } static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs, int32_t index) { - SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false}; + SResultWindowInfo win = { + .pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false}; return taosArrayInsert(pWinInfos, index, &win); } static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs) { - SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false}; + SResultWindowInfo win = { + .pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false}; return taosArrayPush(pWinInfos, &win); } @@ -4147,8 +4121,8 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It } } -static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, - int32_t numOfOutput, SOperatorInfo* pOperator, SHashObj* pStUpdated, bool needCreate) { +static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t numOfOutput, + SOperatorInfo* pOperator, SHashObj* pStUpdated, bool needCreate) { SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -4161,7 +4135,8 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWin uint64_t groupId = pParentWin->groupId; int32_t winIndex = 0; if (needCreate) { - pParentWin = getSessionTimeWindow(&pInfo->streamAggSup, pParentWin->win.skey, pParentWin->win.ekey, groupId, 0, &winIndex); + pParentWin = + getSessionTimeWindow(&pInfo->streamAggSup, pParentWin->win.skey, pParentWin->win.ekey, groupId, 0, &winIndex); } setWindowOutputBuf(pParentWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->streamAggSup, pTaskInfo); @@ -4292,9 +4267,9 @@ static void removeSessionResults(SHashObj* pHashMap, SArray* pWins) { } int32_t compareWinKey(void* pKey, void* data, int32_t index) { - SArray* res = (SArray*)data; + SArray* res = (SArray*)data; SResKeyPos* pos = taosArrayGetP(res, index); - SWinKey* pData = (SWinKey*)pKey; + SWinKey* pData = (SWinKey*)pKey; if (pData->ts == *(int64_t*)pos->key) { if (pData->groupId > pos->groupId) { return 1; @@ -4317,7 +4292,7 @@ static void removeSessionDeleteResults(SArray* update, SHashObj* pStDeleted) { int32_t num = taosArrayGetSize(update); for (int32_t i = 0; i < num; i++) { SResKeyPos* pos = taosArrayGetP(update, i); - SWinKey winKey = {.ts = *(int64_t*)pos->key, .groupId = pos->groupId}; + SWinKey winKey = {.ts = *(int64_t*)pos->key, .groupId = pos->groupId}; taosHashRemove(pStDeleted, &winKey, sizeof(SWinKey)); } } @@ -4346,7 +4321,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); SHashObj* pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK); SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); // SResKeyPos + SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); // SResKeyPos while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -4362,8 +4337,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; - doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, START_TS_COLUMN_INDEX, pChildOp->exprSupp.numOfExprs, - 0, NULL); + doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, START_TS_COLUMN_INDEX, + pChildOp->exprSupp.numOfExprs, 0, NULL); rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, NULL, false); } taosArrayDestroy(pWins); @@ -4464,7 +4439,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } - + { doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); if (pBInfo->pRes->info.rows > 0) { @@ -4856,8 +4831,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl i, &allEqual, pStDeleted); if (!allEqual) { uint64_t uid = 0; - appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey, - &uid, &groupId); + appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey, &uid, &groupId); taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo); continue; @@ -4886,7 +4860,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; SStreamStateAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; - int64_t maxTs = INT64_MIN; + int64_t maxTs = INT64_MIN; if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { @@ -5035,7 +5009,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->info = pInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL, destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); - initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType, pInfo->primaryTsIndex); + initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType, + pInfo->primaryTsIndex); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -5064,8 +5039,8 @@ static int32_t outputMergeAlignedIntervalResult(SOperatorInfo* pOperatorInfo, ui SExprSupp* pSup = &pOperatorInfo->exprSupp; SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &wstartTs, TSDB_KEYSIZE, tableGroupId); - SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, - GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet( + iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); ASSERT(p1 != NULL); finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pSup->pCtx, pSup->pExprInfo, pSup->numOfExprs, @@ -5274,11 +5249,11 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprSupp* pSup = &pOperator->exprSupp; miaInfo->pCondition = pNode->window.node.pConditions; - miaInfo->curTs = INT64_MIN; - iaInfo->win = pTaskInfo->window; - iaInfo->inputOrder = TSDB_ORDER_ASC; - iaInfo->interval = interval; - iaInfo->execModel = pTaskInfo->execModel; + miaInfo->curTs = INT64_MIN; + iaInfo->win = pTaskInfo->window; + iaInfo->inputOrder = TSDB_ORDER_ASC; + iaInfo->interval = interval; + iaInfo->execModel = pTaskInfo->execModel; iaInfo->primaryTsIndex = ((SColumnNode*)pNode->window.pTspk)->slotId; iaInfo->binfo.mergeResultBlock = pNode->window.mergeDataBlock; @@ -5301,12 +5276,12 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, initResultRowInfo(&iaInfo->binfo.resultRowInfo); blockDataEnsureCapacity(iaInfo->binfo.pRes, pOperator->resultInfo.capacity); - pOperator->name = "TimeMergeAlignedIntervalAggOperator"; + pOperator->name = "TimeMergeAlignedIntervalAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = miaInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->pTaskInfo = pTaskInfo; + pOperator->info = miaInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, NULL, destroyMergeAlignedIntervalOperatorInfo, NULL, NULL, NULL); @@ -5359,8 +5334,8 @@ static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t table SExprSupp* pExprSup = &pOperatorInfo->exprSupp; SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &win->skey, TSDB_KEYSIZE, tableGroupId); - SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, - GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet( + iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); ASSERT(p1 != NULL); finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pExprSup->pCtx, pExprSup->pExprInfo, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset, pResultBlock, pTaskInfo); @@ -5585,10 +5560,10 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge pMergeIntervalInfo->groupIntervals = tdListNew(sizeof(SGroupTimeWindow)); SIntervalAggOperatorInfo* pIntervalInfo = &pMergeIntervalInfo->intervalAggOperatorInfo; - pIntervalInfo->win = pTaskInfo->window; + pIntervalInfo->win = pTaskInfo->window; pIntervalInfo->inputOrder = TSDB_ORDER_ASC; - pIntervalInfo->interval = interval; - pIntervalInfo->execModel = pTaskInfo->execModel; + pIntervalInfo->interval = interval; + pIntervalInfo->execModel = pTaskInfo->execModel; pIntervalInfo->binfo.mergeResultBlock = pIntervalPhyNode->window.mergeDataBlock; pIntervalInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; @@ -5605,7 +5580,6 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge initBasicInfo(&pIntervalInfo->binfo, pResBlock); initExecTimeWindowInfo(&pIntervalInfo->twAggSup.timeWindowData, &pIntervalInfo->win); - pIntervalInfo->timeWindowInterpo = timeWindowinterpNeeded(pExprSupp->pCtx, num, pIntervalInfo); if (pIntervalInfo->timeWindowInterpo) { pIntervalInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo)); @@ -5616,12 +5590,12 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge initResultRowInfo(&pIntervalInfo->binfo.resultRowInfo); - pOperator->name = "TimeMergeIntervalAggOperator"; + pOperator->name = "TimeMergeIntervalAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pMergeIntervalInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pMergeIntervalInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL, destroyMergeIntervalOperatorInfo, NULL, NULL, NULL); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 9d4010f60e..a2a45938e4 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -243,6 +243,36 @@ FAIL: return 0; } +int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, int32_t vgSz, + int64_t groupId) { + char* ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, groupId); + SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + + // TODO: get hash function by hashMethod + uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName)); + taosMemoryFree(ctbName); + bool found = false; + // TODO: optimize search + int32_t j; + for (j = 0; j < vgSz; j++) { + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); + ASSERT(pVgInfo->vgId > 0); + if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { + if (streamAddBlockToDispatchMsg(pDataBlock, &pReqs[j]) < 0) { + return -1; + } + if (pReqs[j].blockNum == 0) { + atomic_add_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); + } + pReqs[j].blockNum++; + found = true; + break; + } + } + ASSERT(found); + return 0; +} + int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pData) { int32_t code = -1; int32_t blockNum = taosArrayGetSize(pData->blocks); @@ -317,20 +347,10 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat for (int32_t i = 0; i < blockNum; i++) { SSDataBlock* pDataBlock = taosArrayGet(pData->blocks, i); - char* ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, pDataBlock->info.groupId); - // TODO: get hash function by hashMethod - uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName)); - - taosMemoryFree(ctbName); - - bool found = false; - // TODO: optimize search - int32_t j; - for (j = 0; j < vgSz; j++) { - SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); - ASSERT(pVgInfo->vgId > 0); - if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { + // TODO: do not use broadcast + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + for (int32_t j = 0; j < vgSz; j++) { if (streamAddBlockToDispatchMsg(pDataBlock, &pReqs[j]) < 0) { goto FAIL_SHUFFLE_DISPATCH; } @@ -338,11 +358,13 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat atomic_add_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); } pReqs[j].blockNum++; - found = true; - break; } + continue; + } + + if (streamSearchAndAddBlock(pTask, pReqs, pDataBlock, vgSz, pDataBlock->info.groupId) < 0) { + goto FAIL_SHUFFLE_DISPATCH; } - ASSERT(found); } for (int32_t i = 0; i < vgSz; i++) { From 004a84ac81afcd945a87faf10a191238d040a22e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 2 Sep 2022 16:41:32 +0800 Subject: [PATCH 03/16] fix memory leak --- source/dnode/vnode/src/tq/tq.c | 5 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 3 +- source/libs/executor/src/executor.c | 6 +- source/libs/function/src/builtinsimpl.c | 74 ++++++++++++------------- 4 files changed, 46 insertions(+), 42 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index f8fac97710..f9c2757c37 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -829,8 +829,6 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { tDecoderInit(pCoder, pReq, len); tDecodeDeleteRes(pCoder, pRes); - /*ASSERT(pRes->skey != 0);*/ - /*ASSERT(pRes->ekey != 0);*/ tDecoderClear(pCoder); int32_t sz = taosArrayGetSize(pRes->uidList); @@ -859,6 +857,8 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { colDataAppendNULL(taosArrayGet(pDelBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX), i); } + taosArrayDestroy(pRes->uidList); + void* pIter = NULL; while (1) { pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); @@ -890,6 +890,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { streamTaskInputFail(pTask); } } + blockDataDestroy(pDelBlock); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 96e8e44bc9..2724d4cbfd 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1107,8 +1107,7 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq tDecoderInit(pCoder, pReq, len); tDecodeDeleteRes(pCoder, pRes); - ASSERT(pRes->skey != 0); - ASSERT(pRes->ekey != 0); + ASSERT(taosArrayGetSize(pRes->uidList) == 0 || (pRes->skey != 0 && pRes->ekey != 0)); for (int32_t iUid = 0; iUid < taosArrayGetSize(pRes->uidList); iUid++) { code = tsdbDeleteTableData(pVnode->pTsdb, version, pRes->suid, *(uint64_t *)taosArrayGet(pRes->uidList, iUid), diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index a3e0b2b733..037bf543b1 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -52,7 +52,11 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu // TODO: if a block was set but not consumed, // prevent setting a different type of block pInfo->validBlockIndex = 0; - taosArrayClear(pInfo->pBlockLists); + if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) { + taosArrayClearP(pInfo->pBlockLists, taosMemoryFree); + } else { + taosArrayClear(pInfo->pBlockLists); + } if (type == STREAM_INPUT__MERGED_SUBMIT) { // ASSERT(numOfBlocks > 1); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 7160541c13..dcf58759b0 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -76,11 +76,11 @@ typedef struct STopBotResItem { } STopBotResItem; typedef struct STopBotRes { - int32_t maxSize; - int16_t type; + int32_t maxSize; + int16_t type; - STuplePos nullTuplePos; - bool nullTupleSaved; + STuplePos nullTuplePos; + bool nullTupleSaved; STopBotResItem* pItems; } STopBotRes; @@ -223,14 +223,14 @@ typedef struct SMavgInfo { } SMavgInfo; typedef struct SSampleInfo { - int32_t samples; - int32_t totalPoints; - int32_t numSampled; - uint8_t colType; - int16_t colBytes; + int32_t samples; + int32_t totalPoints; + int32_t numSampled; + uint8_t colType; + int16_t colBytes; - STuplePos nullTuplePos; - bool nullTupleSaved; + STuplePos nullTuplePos; + bool nullTupleSaved; char* data; STuplePos* tuplePos; @@ -1147,7 +1147,7 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { } static STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock); -static int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); +static int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); static const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos); static int32_t findRowIndex(int32_t start, int32_t num, SColumnInfoData* pCol, const char* tval) { @@ -1357,8 +1357,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { numOfElems += 1; } - } else if (type == TSDB_DATA_TYPE_BIGINT || - type == TSDB_DATA_TYPE_TIMESTAMP) { + } else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) { int64_t* pData = (int64_t*)pCol->pData; int64_t* val = (int64_t*)&pBuf->v; @@ -1581,7 +1580,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { } _min_max_over: - if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pBuf->nullTupleSaved ) { + if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pBuf->nullTupleSaved) { pBuf->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock); pBuf->nullTupleSaved = true; } @@ -1601,7 +1600,8 @@ int32_t maxFunction(SqlFunctionCtx* pCtx) { } static void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t rowIndex); -static void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex); +static void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, + int32_t rowIndex); int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); @@ -1651,7 +1651,7 @@ void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuple if (pCtx->saveHandle.pBuf != NULL) { if (pTuplePos->pageId != -1) { - int32_t numOfCols = pCtx->subsidiaries.num; + int32_t numOfCols = pCtx->subsidiaries.num; const char* p = loadTupleData(pCtx, pTuplePos); bool* nullList = (bool*)p; @@ -1660,7 +1660,7 @@ void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuple // todo set the offset value to optimize the performance. for (int32_t j = 0; j < numOfCols; ++j) { SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j]; - int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; + int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); ASSERT(pc->pExpr->base.resSchema.bytes == pDstCol->info.bytes); @@ -1701,7 +1701,7 @@ void appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) char* pData = colDataGetData(pSrcCol, rowIndex); // append to dest col - int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; + int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; SColumnInfoData* pDstCol = taosArrayGet(pCtx->pDstBlock->pDataBlock, dstSlotId); ASSERT(pc->pExpr->base.resSchema.bytes == pDstCol->info.bytes); @@ -1712,7 +1712,6 @@ void appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) colDataAppend(pDstCol, pos, pData, false); } } - } void replaceTupleData(STuplePos* pDestPos, STuplePos* pSourcePos) { @@ -2590,8 +2589,8 @@ static void apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInfo* memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); pHisto->elems = (SHistBin*)((char*)pHisto + sizeof(SHistogramInfo)); - qDebug("%s merge histo, total:%" PRId64 ", entry:%d, %p", __FUNCTION__, pHisto->numOfElems, - pHisto->numOfEntries, pHisto); + qDebug("%s merge histo, total:%" PRId64 ", entry:%d, %p", __FUNCTION__, pHisto->numOfElems, pHisto->numOfEntries, + pHisto); } else { pHisto->elems = (SHistBin*)((char*)pHisto + sizeof(SHistogramInfo)); qDebug("%s input histogram, elem:%" PRId64 ", entry:%d, %p", __FUNCTION__, pHisto->numOfElems, @@ -2601,8 +2600,8 @@ static void apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInfo* memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); pHisto->elems = (SHistBin*)((char*)pHisto + sizeof(SHistogramInfo)); - qDebug("%s merge histo, total:%" PRId64 ", entry:%d, %p", __FUNCTION__, pHisto->numOfElems, - pHisto->numOfEntries, pHisto); + qDebug("%s merge histo, total:%" PRId64 ", entry:%d, %p", __FUNCTION__, pHisto->numOfElems, pHisto->numOfEntries, + pHisto); tHistogramDestroy(&pRes); } } @@ -2629,8 +2628,8 @@ int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) { } if (pInfo->algo != APERCT_ALGO_TDIGEST) { - qDebug("%s after merge, total:%d, numOfEntry:%d, %p", __FUNCTION__, pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, - pInfo->pHisto); + qDebug("%s after merge, total:%d, numOfEntry:%d, %p", __FUNCTION__, pInfo->pHisto->numOfElems, + pInfo->pHisto->numOfEntries, pInfo->pHisto); } SET_VAL(pResInfo, 1, 1); @@ -2709,7 +2708,7 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) } EFuncDataRequired lastDynDataReq(void* pRes, STimeWindow* pTimeWindow) { - SResultRowEntryInfo* pEntry = (SResultRowEntryInfo*) pRes; + SResultRowEntryInfo* pEntry = (SResultRowEntryInfo*)pRes; // not initialized yet, data is required if (pEntry == NULL) { @@ -2752,7 +2751,8 @@ static FORCE_INLINE TSKEY getRowPTs(SColumnInfoData* pTsColInfo, int32_t rowInde return *(TSKEY*)colDataGetData(pTsColInfo, rowIndex); } -static void firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SqlFunctionCtx* pCtx, SFirstLastRes* pInfo) { +static void firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SqlFunctionCtx* pCtx, + SFirstLastRes* pInfo) { if (pCtx->subsidiaries.num <= 0) { return; } @@ -3176,7 +3176,7 @@ bool diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { static void doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv) { switch (type) { case TSDB_DATA_TYPE_BOOL: - pDiffInfo->prev.i64 = *(bool*)pv? 1:0; + pDiffInfo->prev.i64 = *(bool*)pv ? 1 : 0; break; case TSDB_DATA_TYPE_TINYINT: pDiffInfo->prev.i64 = *(int8_t*)pv; @@ -3537,7 +3537,8 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData * |(n columns, one bit for each column)| src column #1| src column #2| * +------------------------------------+--------------+--------------+ */ -void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsidiaryResInfo* pSubsidiaryies, char* buf) { +void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsidiaryResInfo* pSubsidiaryies, + char* buf) { char* nullList = buf; char* pStart = (char*)(nullList + sizeof(bool) * pSubsidiaryies->num); @@ -3585,7 +3586,7 @@ static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf } } - p = (STuplePos) {.pageId = pHandle->currentPage, .offset = pPage->num}; + p = (STuplePos){.pageId = pHandle->currentPage, .offset = pPage->num}; memcpy(pPage->data + pPage->num, pBuf, length); pPage->num += length; @@ -3621,7 +3622,6 @@ static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); } else { - } return TSDB_CODE_SUCCESS; @@ -3636,7 +3636,7 @@ static int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSD static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos) { if (pHandle->pBuf != NULL) { SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId); - char* p = pPage->data + pPos->offset; + char* p = pPage->data + pPos->offset; releaseBufPage(pHandle->pBuf, pPage); return p; } else { @@ -3980,8 +3980,8 @@ int32_t elapsedFunction(SqlFunctionCtx* pCtx) { } if (pCtx->end.key == INT64_MIN) { - pInfo->min = (pInfo->min > ptsList[start + pInput->numOfRows - 1]) ? - ptsList[start + pInput->numOfRows - 1] : pInfo->min; + pInfo->min = + (pInfo->min > ptsList[start + pInput->numOfRows - 1]) ? ptsList[start + pInput->numOfRows - 1] : pInfo->min; } else { pInfo->min = pCtx->end.key; } @@ -3993,8 +3993,8 @@ int32_t elapsedFunction(SqlFunctionCtx* pCtx) { } if (pCtx->end.key == INT64_MIN) { - pInfo->max = (pInfo->max < ptsList[start + pInput->numOfRows - 1]) ? - ptsList[start + pInput->numOfRows - 1] : pInfo->max; + pInfo->max = + (pInfo->max < ptsList[start + pInput->numOfRows - 1]) ? ptsList[start + pInput->numOfRows - 1] : pInfo->max; } else { pInfo->max = pCtx->end.key + 1; } From d94a758310f4d506b9d4bc650989b47e2fb89267 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 2 Sep 2022 16:48:19 +0800 Subject: [PATCH 04/16] fix: sleep 1s before starting query --- tests/script/tsim/valgrind/checkError6.sim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index d85a1bebc8..11a387ed4d 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -158,6 +158,8 @@ print =============== restart system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -v +sleep 1000 + sql select avg(tbcol) as c from stb sql select avg(tbcol) as c from stb where ts <= 1601481840000 sql select avg(tbcol) as c from stb where tgcol < 5 and ts <= 1601481840000 From 6603a2eda2c52ba78f95d11128862a759dd7a4f7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Sep 2022 16:54:49 +0800 Subject: [PATCH 05/16] enh: show precision of db if user no sysinfo privilege --- source/common/src/systable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index fdacfa55b8..3b6de2e399 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -90,7 +90,7 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, - {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "retentions", .bytes = 60 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL, .sysInfo = true}, From 86348e74392c1b290747eab79471059d1cbd978f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Sep 2022 17:27:47 +0800 Subject: [PATCH 06/16] fix(query): initialize the last block reader successfully. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 7f492f73b0..7133ec1717 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -353,12 +353,13 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) { int32_t step = asc ? 1 : -1; pIter->index += step; - pIter->pLastBlockReader->uid = 0; - tMergeTreeClose(&pIter->pLastBlockReader->mergeTree); if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) { return false; } + pIter->pLastBlockReader->uid = 0; + tMergeTreeClose(&pIter->pLastBlockReader->mergeTree); + // check file the time range of coverage STimeWindow win = {0}; @@ -2161,8 +2162,6 @@ _err: static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}; - - initMemDataIterator(pScanInfo, pReader); TSDBROW* pRow = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader); if (pRow != NULL) { key = TSDBROW_KEY(pRow); @@ -2356,6 +2355,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { pBlock = getCurrentBlock(pBlockIter); } + initLastBlockReader(pLastBlockReader, pScanInfo, pReader); TSDBKEY key = getCurrentKeyInBuf(pScanInfo, pReader); if (pBlockInfo == NULL) { // build data block from last data file From 4afdd6f7a1fa28f93cbcc64176063cabe02916e5 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 2 Sep 2022 18:27:59 +0800 Subject: [PATCH 07/16] feat: update taostools a4d9b92 for3.0 (#16624) * feat: update taos-tools f169c0f for 3.0 * feat: update taos-tools a4d9b92 for 3.0 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 3c7067a9f8..b905e30489 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG f169c0f + GIT_TAG a4d9b92 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From ad49f13159e31bfb04d796db33083994d5ed4d7d Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 2 Sep 2022 18:53:31 +0800 Subject: [PATCH 08/16] fix(stream):min max function error --- source/libs/executor/src/timewindowoperator.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index ab28c91867..5a056754eb 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3166,6 +3166,12 @@ static void addRetriveWindow(SArray* wins, SStreamFinalIntervalOperatorInfo* pIn } } +static void clearFunctionContext(SExprSupp* pSup) { + for (int32_t i = 0; i < pSup->numOfExprs; i++) { + pSup->pCtx[i].saveHandle.currentPage = -1; + } +} + static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; @@ -3201,6 +3207,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { if (pInfo->binfo.pRes->info.rows == 0) { pOperator->status = OP_EXEC_DONE; if (!IS_FINAL_OP(pInfo)) { + clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer clearStreamIntervalOperator(pInfo); qDebug("===stream===clear semi operator"); @@ -4461,6 +4468,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } if (pOperator->status == OP_RES_TO_RETURN) { + clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer clearStreamSessionOperator(pInfo); pOperator->status = OP_EXEC_DONE; From 25c67f6ebbf20a1bee5232b36e8a1e3416712836 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Sep 2022 17:12:08 +0800 Subject: [PATCH 09/16] enh: add hash prefix and suffix option to db --- include/common/tmsg.h | 4 ++++ include/util/tdef.h | 6 ++++++ include/util/tutil.h | 3 +++ source/common/src/tmsg.c | 8 ++++++++ source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 6 +++++- source/dnode/mnode/impl/inc/mndDef.h | 6 ++++-- source/dnode/mnode/impl/src/mndDb.c | 12 +++++++++++- source/dnode/mnode/impl/src/mndVgroup.c | 2 ++ source/dnode/vnode/inc/vnode.h | 2 ++ source/dnode/vnode/src/vnd/vnodeCfg.c | 10 +++++++++- 10 files changed, 54 insertions(+), 5 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 879f8c6928..8efd58739d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -785,6 +785,8 @@ typedef struct { int32_t walRollPeriod; int64_t walSegmentSize; int32_t sstTrigger; + int16_t hashPrefix; + int16_t hashSuffix; } SCreateDbReq; int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq); @@ -1193,6 +1195,8 @@ typedef struct { int32_t walRollPeriod; int64_t walSegmentSize; int16_t sstTrigger; + int16_t hashPrefix; + int16_t hashSuffix; } SCreateVnodeReq; int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); diff --git a/include/util/tdef.h b/include/util/tdef.h index 48f37fca28..57fde32bed 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -362,6 +362,12 @@ typedef enum ELogicConditionType { #define TSDB_MIN_SST_TRIGGER 1 #define TSDB_MAX_SST_TRIGGER 128 #define TSDB_DEFAULT_SST_TRIGGER 8 +#define TSDB_MIN_HASH_PREFIX 0 +#define TSDB_MAX_HASH_PREFIX 128 +#define TSDB_DEFAULT_HASH_PREFIX 0 +#define TSDB_MIN_HASH_SUFFIX 0 +#define TSDB_MAX_HASH_SUFFIX 128 +#define TSDB_DEFAULT_HASH_SUFFIX 0 #define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1 #define TSDB_REP_DEF_DB_WAL_RET_PERIOD 0 diff --git a/include/util/tutil.h b/include/util/tutil.h index 6a1a40f14c..a0c7b3d7ad 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -20,6 +20,7 @@ #include "tcrc32c.h" #include "tdef.h" #include "tmd5.h" +#include "thash.h" #ifdef __cplusplus extern "C" { @@ -68,6 +69,8 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar memcpy(target, buf, TSDB_PASSWORD_LEN); } +#define taosGetTbHashVal(tbname, tblen, method, prefix, suffix) MurmurHash3_32((tbname), (tblen)) + #ifdef __cplusplus } #endif diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 9947d0e3b2..9b6737a210 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2025,6 +2025,8 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { if (tEncodeI32(&encoder, pReq->walRollPeriod) < 0) return -1; if (tEncodeI64(&encoder, pReq->walSegmentSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->sstTrigger) < 0) return -1; + if (tEncodeI16(&encoder, pReq->hashPrefix) < 0) return -1; + if (tEncodeI16(&encoder, pReq->hashSuffix) < 0) return -1; if (tEncodeI8(&encoder, pReq->ignoreExist) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1; for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { @@ -2072,6 +2074,8 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) if (tDecodeI32(&decoder, &pReq->walRollPeriod) < 0) return -1; if (tDecodeI64(&decoder, &pReq->walSegmentSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->sstTrigger) < 0) return -1; + if (tDecodeI16(&decoder, &pReq->hashPrefix) < 0) return -1; + if (tDecodeI16(&decoder, &pReq->hashSuffix) < 0) return -1; if (tDecodeI8(&decoder, &pReq->ignoreExist) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfRetensions) < 0) return -1; pReq->pRetensions = taosArrayInit(pReq->numOfRetensions, sizeof(SRetention)); @@ -3767,6 +3771,8 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI32(&encoder, pReq->walRollPeriod) < 0) return -1; if (tEncodeI64(&encoder, pReq->walSegmentSize) < 0) return -1; if (tEncodeI16(&encoder, pReq->sstTrigger) < 0) return -1; + if (tEncodeI16(&encoder, pReq->hashPrefix) < 0) return -1; + if (tEncodeI16(&encoder, pReq->hashSuffix) < 0) return -1; tEndEncode(&encoder); @@ -3840,6 +3846,8 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI32(&decoder, &pReq->walRollPeriod) < 0) return -1; if (tDecodeI64(&decoder, &pReq->walSegmentSize) < 0) return -1; if (tDecodeI16(&decoder, &pReq->sstTrigger) < 0) return -1; + if (tDecodeI16(&decoder, &pReq->hashPrefix) < 0) return -1; + if (tDecodeI16(&decoder, &pReq->hashSuffix) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index c684683ee9..830bf9523a 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -171,6 +171,8 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->hashBegin = pCreate->hashBegin; pCfg->hashEnd = pCreate->hashEnd; pCfg->hashMethod = pCreate->hashMethod; + pCfg->hashPrefix = pCreate->hashPrefix; + pCfg->hashSuffix = pCreate->hashSuffix; pCfg->standby = pCfg->standby; pCfg->syncCfg.myIndex = pCreate->selfIndex; @@ -220,9 +222,11 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dDebug("vgId:%d, start to create vnode, tsma:%d standby:%d cacheLast:%d cacheLastSize:%d sstTrigger:%d", + dInfo("vgId:%d, start to create vnode, tsma:%d standby:%d cacheLast:%d cacheLastSize:%d sstTrigger:%d", createReq.vgId, createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize, createReq.sstTrigger); + dInfo("vgId:%d, hashMethod:%d begin:%u end:%u prefix:%d surfix:%d", createReq.vgId, createReq.hashMethod, + createReq.hashBegin, createReq.hashEnd, createReq.hashPrefix, createReq.hashSuffix); vmGenerateVnodeCfg(&createReq, &vnodeCfg); if (vmTsmaAdjustDays(&vnodeCfg, &createReq) < 0) { diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 179e7f4448..2268a18455 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -305,13 +305,15 @@ typedef struct { int8_t hashMethod; // default is 1 int8_t cacheLast; int8_t schemaless; + int16_t hashPrefix; + int16_t hashSuffix; + int16_t sstTrigger; int32_t numOfRetensions; SArray* pRetensions; int32_t walRetentionPeriod; - int64_t walRetentionSize; int32_t walRollPeriod; + int64_t walRetentionSize; int64_t walSegmentSize; - int16_t sstTrigger; } SDbCfg; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index d1313d5a17..d808dfbbf3 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -30,7 +30,7 @@ #include "systable.h" #define DB_VER_NUMBER 1 -#define DB_RESERVE_SIZE 62 +#define DB_RESERVE_SIZE 58 static SSdbRaw *mndDbActionEncode(SDbObj *pDb); static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); @@ -125,6 +125,8 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT32(pRaw, dataPos, pDb->cfg.walRollPeriod, _OVER) SDB_SET_INT64(pRaw, dataPos, pDb->cfg.walSegmentSize, _OVER) SDB_SET_INT16(pRaw, dataPos, pDb->cfg.sstTrigger, _OVER) + SDB_SET_INT16(pRaw, dataPos, pDb->cfg.hashPrefix, _OVER) + SDB_SET_INT16(pRaw, dataPos, pDb->cfg.hashSuffix, _OVER) SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) @@ -209,6 +211,8 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.walRollPeriod, _OVER) SDB_GET_INT64(pRaw, dataPos, &pDb->cfg.walSegmentSize, _OVER) SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.sstTrigger, _OVER) + SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.hashPrefix, _OVER) + SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.hashSuffix, _OVER) SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) taosInitRWLatch(&pDb->lock); @@ -334,6 +338,8 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->walRollPeriod < TSDB_DB_MIN_WAL_ROLL_PERIOD) return -1; if (pCfg->walSegmentSize < TSDB_DB_MIN_WAL_SEGMENT_SIZE) return -1; if (pCfg->sstTrigger < TSDB_MIN_SST_TRIGGER || pCfg->sstTrigger > TSDB_MAX_SST_TRIGGER) return -1; + if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1; + if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1; terrno = 0; return terrno; @@ -368,6 +374,8 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->walRollPeriod < 0) pCfg->walRollPeriod = TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD; if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE; if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER; + if (pCfg->hashPrefix < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX; + if (pCfg->hashSuffix < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX; } static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { @@ -485,6 +493,8 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, .walRollPeriod = pCreate->walRollPeriod, .walSegmentSize = pCreate->walSegmentSize, .sstTrigger = pCreate->sstTrigger, + .hashPrefix = pCreate->hashPrefix, + .hashSuffix = pCreate->hashSuffix, }; dbObj.cfg.numOfRetensions = pCreate->numOfRetensions; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index f54e831b0b..d5f9f8a5cf 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -235,6 +235,8 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.walRollPeriod = pDb->cfg.walRollPeriod; createReq.walSegmentSize = pDb->cfg.walSegmentSize; createReq.sstTrigger = pDb->cfg.sstTrigger; + createReq.hashPrefix = pDb->cfg.hashPrefix; + createReq.hashSuffix = pDb->cfg.hashSuffix; for (int32_t v = 0; v < pVgroup->replica; ++v) { SReplica *pReplica = &createReq.replicas[v]; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index a1c2265c2e..0fea96764e 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -288,6 +288,8 @@ struct SVnodeCfg { uint32_t hashBegin; uint32_t hashEnd; int16_t sstTrigger; + int16_t hashPrefix; + int16_t hashSuffix; }; typedef struct { diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 24dda29330..f379d9f71a 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -14,6 +14,7 @@ */ #include "vnd.h" +#include "tutil.h" const SVnodeCfg vnodeCfgDefault = {.vgId = -1, .dbname = "", @@ -110,6 +111,8 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "hashPrefix", pCfg->hashPrefix) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1; @@ -214,6 +217,10 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (code < 0) return -1; tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod, code); if (code < 0) return -1; + tjsonGetNumberValue(pJson, "hashPrefix", pCfg->hashPrefix, code); + if (code < 0) return -1; + tjsonGetNumberValue(pJson, "hashSuffix", pCfg->hashSuffix, code); + if (code < 0) return -1; tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum, code); if (code < 0) return -1; @@ -250,7 +257,8 @@ int vnodeValidateTableHash(SVnode *pVnode, char *tableFName) { switch (pVnode->config.hashMethod) { default: - hashValue = MurmurHash3_32(tableFName, strlen(tableFName)); + hashValue = taosGetTbHashVal(tableFName, strlen(tableFName), pVnode->config.hashMethod, pVnode->config.hashPrefix, + pVnode->config.hashSuffix); break; } From 527f0ad8e4bc96f18b2c752ce22c1eef9f7b141c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 2 Sep 2022 20:54:04 +0800 Subject: [PATCH 10/16] fix:hash obj reuse after free --- source/client/src/clientSml.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index c6ba2d6892..f2493f6c57 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -547,6 +547,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { goto end; } needCheckMeta = true; + taosHashCleanup(hashTmp); + hashTmp = NULL; } else { uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code)); goto end; @@ -576,7 +578,6 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { sTableData->tableMeta = pTableMeta; tableMetaSml = (SSmlSTableMeta **)taosHashIterate(info->superTables, tableMetaSml); - taosHashCleanup(hashTmp); } return 0; From c984fcaee0084fb9417b5789f671d232b28d791f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 3 Sep 2022 07:09:13 +0800 Subject: [PATCH 11/16] chore: move few tests items out (#16637) * chore: move few tests items out * fix: add taos as dep to tmq_demo --- CMakeLists.txt | 3 +- tests/test/c/CMakeLists.txt | 1 + utils/CMakeLists.txt | 4 + utils/test/c/CMakeLists.txt | 59 ++ utils/test/c/createTable.c | 471 +++++++++++ utils/test/c/sdbDump.c | 475 +++++++++++ utils/test/c/sml_test.c | 1191 +++++++++++++++++++++++++++ utils/test/c/tmqDemo.c | 705 ++++++++++++++++ utils/test/c/tmqSim.c | 1520 +++++++++++++++++++++++++++++++++++ utils/test/c/tmq_taosx_ci.c | 623 ++++++++++++++ utils/tsim/CMakeLists.txt | 14 + utils/tsim/inc/simInt.h | 186 +++++ utils/tsim/inc/simParse.h | 57 ++ utils/tsim/src/simExe.c | 1016 +++++++++++++++++++++++ utils/tsim/src/simMain.c | 76 ++ utils/tsim/src/simParse.c | 1097 +++++++++++++++++++++++++ utils/tsim/src/simSystem.c | 145 ++++ 17 files changed, 7642 insertions(+), 1 deletion(-) create mode 100644 utils/CMakeLists.txt create mode 100644 utils/test/c/CMakeLists.txt create mode 100644 utils/test/c/createTable.c create mode 100644 utils/test/c/sdbDump.c create mode 100644 utils/test/c/sml_test.c create mode 100644 utils/test/c/tmqDemo.c create mode 100644 utils/test/c/tmqSim.c create mode 100644 utils/test/c/tmq_taosx_ci.c create mode 100644 utils/tsim/CMakeLists.txt create mode 100644 utils/tsim/inc/simInt.h create mode 100644 utils/tsim/inc/simParse.h create mode 100644 utils/tsim/src/simExe.c create mode 100644 utils/tsim/src/simMain.c create mode 100644 utils/tsim/src/simParse.c create mode 100644 utils/tsim/src/simSystem.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c7e31bfbb..e534f925aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,7 +34,8 @@ endif(${BUILD_TEST}) add_subdirectory(source) add_subdirectory(tools) -add_subdirectory(tests) +#add_subdirectory(tests) +add_subdirectory(utils) add_subdirectory(examples/c) # docs diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 31331b5265..839c65b633 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -1,4 +1,5 @@ add_executable(tmq_demo tmqDemo.c) +add_dependencies(tmq_demo taos) add_executable(tmq_sim tmqSim.c) add_executable(create_table createTable.c) add_executable(tmq_taosx_ci tmq_taosx_ci.c) diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt new file mode 100644 index 0000000000..59cbbb3147 --- /dev/null +++ b/utils/CMakeLists.txt @@ -0,0 +1,4 @@ +#ADD_SUBDIRECTORY(examples/c) +ADD_SUBDIRECTORY(tsim) +ADD_SUBDIRECTORY(test/c) +#ADD_SUBDIRECTORY(comparisonTest/tdengine) diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt new file mode 100644 index 0000000000..839c65b633 --- /dev/null +++ b/utils/test/c/CMakeLists.txt @@ -0,0 +1,59 @@ +add_executable(tmq_demo tmqDemo.c) +add_dependencies(tmq_demo taos) +add_executable(tmq_sim tmqSim.c) +add_executable(create_table createTable.c) +add_executable(tmq_taosx_ci tmq_taosx_ci.c) +add_executable(sml_test sml_test.c) +target_link_libraries( + create_table + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os +) +target_link_libraries( + tmq_demo + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os +) +target_link_libraries( + tmq_sim + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os +) +target_link_libraries( + tmq_taosx_ci + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os +) + +target_link_libraries( + sml_test + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os +) + +add_executable(sdbDump sdbDump.c) +target_link_libraries( + sdbDump + PUBLIC dnode + PUBLIC mnode + PUBLIC stream + PUBLIC sdb + PUBLIC os +) +target_include_directories( + sdbDump + PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode" + PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/impl/inc" + PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/sdb/inc" + PRIVATE "${TD_SOURCE_DIR}/source/dnode/mgmt/node_mgmt/inc" +) diff --git a/utils/test/c/createTable.c b/utils/test/c/createTable.c new file mode 100644 index 0000000000..6a0f8e244e --- /dev/null +++ b/utils/test/c/createTable.c @@ -0,0 +1,471 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taos.h" +#include "taoserror.h" +#include "tlog.h" + +#define GREEN "\033[1;32m" +#define NC "\033[0m" + +char dbName[32] = "db"; +char stbName[64] = "st"; +int32_t numOfThreads = 1; +int64_t numOfTables = 200000; +int64_t startOffset = 0; +int32_t createTable = 1; +int32_t insertData = 0; +int32_t batchNumOfTbl = 100; +int32_t batchNumOfRow = 1; +int32_t totalRowsOfPerTbl = 1; +int32_t numOfVgroups = 2; +int32_t showTablesFlag = 0; +int32_t queryFlag = 0; + +int64_t startTimestamp = 1640966400000; // 2020-01-01 00:00:00.000 + +typedef struct { + int64_t tableBeginIndex; + int64_t tableEndIndex; + int32_t threadIndex; + char dbName[32]; + char stbName[64]; + float createTableSpeed; + float insertDataSpeed; + int64_t startMs; + int64_t maxDelay; + int64_t minDelay; + TdThread thread; +} SThreadInfo; + +// void parseArgument(int32_t argc, char *argv[]); +// void *threadFunc(void *param); +// void createDbAndStb(); + +void createDbAndStb() { + pPrint("start to create db and stable"); + char qstr[64000]; + + TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("failed to connect to DB, reason:%s", taos_errstr(NULL)); + exit(1); + } + + sprintf(qstr, "create database if not exists %s vgroups %d", dbName, numOfVgroups); + TAOS_RES *pRes = taos_query(con, qstr); + int32_t code = taos_errno(pRes); + if (code != 0) { + pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(pRes), + taos_errstr(pRes)); + exit(0); + } + taos_free_result(pRes); + + sprintf(qstr, "use %s", dbName); + pRes = taos_query(con, qstr); + code = taos_errno(pRes); + if (code != 0) { + pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); + exit(0); + } + taos_free_result(pRes); + + sprintf(qstr, "create table if not exists %s (ts timestamp, i int) tags (j bigint)", stbName); + pRes = taos_query(con, qstr); + code = taos_errno(pRes); + if (code != 0) { + pError("failed to create stable, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); + exit(0); + } + taos_free_result(pRes); + + taos_close(con); +} + +void printCreateProgress(SThreadInfo *pInfo, int64_t t) { + int64_t endMs = taosGetTimestampMs(); + int64_t totalTables = t - pInfo->tableBeginIndex; + float seconds = (endMs - pInfo->startMs) / 1000.0; + float speed = totalTables / seconds; + pInfo->createTableSpeed = speed; + pPrint("thread:%d, %" PRId64 " tables created, time:%.2f sec, speed:%.1f tables/second, ", pInfo->threadIndex, + totalTables, seconds, speed); +} + +void printInsertProgress(SThreadInfo *pInfo, int64_t insertTotalRows) { + int64_t endMs = taosGetTimestampMs(); + //int64_t totalTables = t - pInfo->tableBeginIndex; + float seconds = (endMs - pInfo->startMs) / 1000.0; + float speed = insertTotalRows / seconds; + pInfo->insertDataSpeed = speed; + pPrint("thread:%d, %" PRId64 " rows inserted, time:%.2f sec, speed:%.1f rows/second, ", pInfo->threadIndex, + insertTotalRows, seconds, speed); +} + +static int64_t getResult(TAOS_RES *tres) { + TAOS_ROW row = taos_fetch_row(tres); + if (row == NULL) { + return 0; + } + + int num_fields = taos_num_fields(tres); + TAOS_FIELD *fields = taos_fetch_fields(tres); + int precision = taos_result_precision(tres); + + int64_t numOfRows = 0; + do { + numOfRows++; + row = taos_fetch_row(tres); + } while (row != NULL); + + return numOfRows; +} + +void showTables() { + pPrint("start to show tables"); + char qstr[128]; + + TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("failed to connect to DB, reason:%s", taos_errstr(NULL)); + exit(1); + } + + snprintf(qstr, 128, "use %s", dbName); + TAOS_RES *pRes = taos_query(con, qstr); + int code = taos_errno(pRes); + if (code != 0) { + pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); + exit(1); + } + taos_free_result(pRes); + + sprintf(qstr, "show tables"); + pRes = taos_query(con, qstr); + code = taos_errno(pRes); + if (code != 0) { + pError("failed to show tables, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); + exit(0); + } + + int64_t totalTableNum = getResult(pRes); + taos_free_result(pRes); + + pPrint("%s database: %s, total %" PRId64 " tables %s", GREEN, dbName, totalTableNum, NC); + + taos_close(con); +} + +void *threadFunc(void *param) { + SThreadInfo *pInfo = (SThreadInfo *)param; + char *qstr = taosMemoryMalloc(batchNumOfTbl * batchNumOfRow * 128); + int32_t code = 0; + + TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(NULL)); + exit(1); + } + + //pPrint("====before thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, + // pInfo->tableEndIndex); + + pInfo->tableBeginIndex += startOffset; + pInfo->tableEndIndex += startOffset; + + pPrint("====thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, pInfo->tableEndIndex); + + sprintf(qstr, "use %s", pInfo->dbName); + TAOS_RES *pRes = taos_query(con, qstr); + taos_free_result(pRes); + + if (createTable) { + int64_t curMs = 0; + int64_t beginMs = taosGetTimestampMs(); + pInfo->startMs = beginMs; + int64_t t = pInfo->tableBeginIndex; + for (; t <= pInfo->tableEndIndex;) { + // int64_t batch = (pInfo->tableEndIndex - t); + // batch = MIN(batch, batchNum); + + int32_t len = sprintf(qstr, "create table"); + for (int32_t i = 0; i < batchNumOfTbl;) { + len += sprintf(qstr + len, " %s_t%" PRId64 " using %s tags(%" PRId64 ")", stbName, t, stbName, t); + t++; + i++; + if (t > pInfo->tableEndIndex) { + break; + } + } + + int64_t startTs = taosGetTimestampUs(); + TAOS_RES *pRes = taos_query(con, qstr); + code = taos_errno(pRes); + if (code != 0) { + pError("failed to create table reason:%s, sql: %s", tstrerror(code), qstr); + } + taos_free_result(pRes); + int64_t endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + // printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay); + if (delay > pInfo->maxDelay) pInfo->maxDelay = delay; + if (delay < pInfo->minDelay) pInfo->minDelay = delay; + + curMs = taosGetTimestampMs(); + if (curMs - beginMs > 10000) { + beginMs = curMs; + // printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t); + printCreateProgress(pInfo, t); + } + } + printCreateProgress(pInfo, t); + } + + if (insertData) { + int64_t insertTotalRows = 0; + int64_t curMs = 0; + int64_t beginMs = taosGetTimestampMs(); + pInfo->startMs = beginMs; + int64_t t = pInfo->tableBeginIndex; + for (; t <= pInfo->tableEndIndex; t++) { + //printf("table name: %"PRId64"\n", t); + int64_t ts = startTimestamp; + for (int32_t i = 0; i < totalRowsOfPerTbl;) { + int32_t len = sprintf(qstr, "insert into "); + len += sprintf(qstr + len, "%s_t%" PRId64 " values ", stbName, t); + for (int32_t j = 0; j < batchNumOfRow; j++) { + len += sprintf(qstr + len, "(%" PRId64 ", 6666) ", ts++); + i++; + insertTotalRows++; + if (i >= totalRowsOfPerTbl) { + break; + } + } + + #if 1 + int64_t startTs = taosGetTimestampUs(); + TAOS_RES *pRes = taos_query(con, qstr); + code = taos_errno(pRes); + if (code != 0) { + pError("failed to insert %s_t%" PRId64 ", reason:%s", stbName, t, tstrerror(code)); + } + taos_free_result(pRes); + int64_t endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + // printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay); + if (delay > pInfo->maxDelay) pInfo->maxDelay = delay; + if (delay < pInfo->minDelay) pInfo->minDelay = delay; + + curMs = taosGetTimestampMs(); + if (curMs - beginMs > 10000) { + beginMs = curMs; + // printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t); + printInsertProgress(pInfo, insertTotalRows); + } + #endif + } + } + printInsertProgress(pInfo, insertTotalRows); + } + + taos_close(con); + taosMemoryFree(qstr); + return 0; +} + +void printHelp() { + char indent[10] = " "; + printf("Used to test the performance while create table\n"); + + printf("%s%s\n", indent, "-c"); + printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); + printf("%s%s\n", indent, "-d"); + printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", stbName); + printf("%s%s\n", indent, "-t"); + printf("%s%s%s%d\n", indent, indent, "numOfThreads, default is ", numOfThreads); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "numOfTables, default is ", numOfTables); + printf("%s%s\n", indent, "-g"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "startOffset, default is ", startOffset); + printf("%s%s\n", indent, "-v"); + printf("%s%s%s%d\n", indent, indent, "numOfVgroups, default is ", numOfVgroups); + printf("%s%s\n", indent, "-a"); + printf("%s%s%s%d\n", indent, indent, "createTable, default is ", createTable); + printf("%s%s\n", indent, "-i"); + printf("%s%s%s%d\n", indent, indent, "insertData, default is ", insertData); + printf("%s%s\n", indent, "-b"); + printf("%s%s%s%d\n", indent, indent, "batchNumOfTbl, default is ", batchNumOfTbl); + printf("%s%s\n", indent, "-w"); + printf("%s%s%s%d\n", indent, indent, "showTablesFlag, default is ", showTablesFlag); + printf("%s%s\n", indent, "-q"); + printf("%s%s%s%d\n", indent, indent, "queryFlag, default is ", queryFlag); + printf("%s%s\n", indent, "-l"); + printf("%s%s%s%d\n", indent, indent, "batchNumOfRow, default is ", batchNumOfRow); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%d\n", indent, indent, "totalRowsOfPerTbl, default is ", totalRowsOfPerTbl); + + exit(EXIT_SUCCESS); +} + +void parseArgument(int32_t argc, char *argv[]) { + for (int32_t i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-d") == 0) { + strcpy(dbName, argv[++i]); + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + strcpy(stbName, argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + numOfTables = atoll(argv[++i]); + } else if (strcmp(argv[i], "-g") == 0) { + startOffset = atoll(argv[++i]); + } else if (strcmp(argv[i], "-v") == 0) { + numOfVgroups = atoi(argv[++i]); + } else if (strcmp(argv[i], "-a") == 0) { + createTable = atoi(argv[++i]); + } else if (strcmp(argv[i], "-i") == 0) { + insertData = atoi(argv[++i]); + } else if (strcmp(argv[i], "-b") == 0) { + batchNumOfTbl = atoi(argv[++i]); + } else if (strcmp(argv[i], "-l") == 0) { + batchNumOfRow = atoi(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + totalRowsOfPerTbl = atoi(argv[++i]); + } else if (strcmp(argv[i], "-w") == 0) { + showTablesFlag = atoi(argv[++i]); + } else if (strcmp(argv[i], "-q") == 0) { + queryFlag = atoi(argv[++i]); + } else { + pPrint("%s unknow para: %s %s", GREEN, argv[++i], NC); + } + } + + pPrint("%s dbName:%s %s", GREEN, dbName, NC); + pPrint("%s stbName:%s %s", GREEN, stbName, NC); + pPrint("%s configDir:%s %s", GREEN, configDir, NC); + pPrint("%s numOfTables:%" PRId64 " %s", GREEN, numOfTables, NC); + pPrint("%s startOffset:%" PRId64 " %s", GREEN, startOffset, NC); + pPrint("%s numOfThreads:%d %s", GREEN, numOfThreads, NC); + pPrint("%s numOfVgroups:%d %s", GREEN, numOfVgroups, NC); + pPrint("%s createTable:%d %s", GREEN, createTable, NC); + pPrint("%s insertData:%d %s", GREEN, insertData, NC); + pPrint("%s batchNumOfTbl:%d %s", GREEN, batchNumOfTbl, NC); + pPrint("%s batchNumOfRow:%d %s", GREEN, batchNumOfRow, NC); + pPrint("%s totalRowsOfPerTbl:%d %s", GREEN, totalRowsOfPerTbl, NC); + pPrint("%s showTablesFlag:%d %s", GREEN, showTablesFlag, NC); + pPrint("%s queryFlag:%d %s", GREEN, queryFlag, NC); + + pPrint("%s start create table performace test %s", GREEN, NC); +} + +int32_t main(int32_t argc, char *argv[]) { + parseArgument(argc, argv); + + if (showTablesFlag) { + showTables(); + return 0; + } + + if (queryFlag) { + // selectRowsFromTable(); + return 0; + } + + if (createTable) { + createDbAndStb(); + } + + pPrint("%d threads are spawned to create %" PRId64 " tables, offset is %" PRId64 " ", numOfThreads, numOfTables, + startOffset); + + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + SThreadInfo *pInfo = (SThreadInfo *)taosMemoryCalloc(numOfThreads, sizeof(SThreadInfo)); + + // int64_t numOfTablesPerThread = numOfTables / numOfThreads; + // numOfTables = numOfTablesPerThread * numOfThreads; + + if (numOfThreads < 1) { + numOfThreads = 1; + } + + int64_t a = numOfTables / numOfThreads; + if (a < 1) { + numOfThreads = numOfTables; + a = 1; + } + + int64_t b = 0; + b = numOfTables % numOfThreads; + + int64_t tableFrom = 0; + for (int32_t i = 0; i < numOfThreads; ++i) { + pInfo[i].tableBeginIndex = tableFrom; + pInfo[i].tableEndIndex = (i < b ? tableFrom + a : tableFrom + a - 1); + tableFrom = pInfo[i].tableEndIndex + 1; + pInfo[i].threadIndex = i; + pInfo[i].minDelay = INT64_MAX; + strcpy(pInfo[i].dbName, dbName); + strcpy(pInfo[i].stbName, stbName); + taosThreadCreate(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); + } + + taosMsleep(300); + for (int32_t i = 0; i < numOfThreads; i++) { + taosThreadJoin(pInfo[i].thread, NULL); + taosThreadClear(&pInfo[i].thread); + } + + int64_t maxDelay = 0; + int64_t minDelay = INT64_MAX; + + float createTableSpeed = 0; + for (int32_t i = 0; i < numOfThreads; ++i) { + createTableSpeed += pInfo[i].createTableSpeed; + + if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay; + if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay; + } + + float insertDataSpeed = 0; + for (int32_t i = 0; i < numOfThreads; ++i) { + insertDataSpeed += pInfo[i].insertDataSpeed; + } + + if (createTable) { + pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 + "us %s", + GREEN, numOfTables, createTableSpeed, numOfThreads, maxDelay, minDelay, NC); + } + + if (insertData) { + pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, + numOfThreads, NC); + } + + taosThreadAttrDestroy(&thattr); + taosMemoryFree(pInfo); +} diff --git a/utils/test/c/sdbDump.c b/utils/test/c/sdbDump.c new file mode 100644 index 0000000000..b90b781e44 --- /dev/null +++ b/utils/test/c/sdbDump.c @@ -0,0 +1,475 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmMgmt.h" +#include "mndInt.h" +#include "sdb.h" +#include "tconfig.h" +#include "tjson.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-result" + +#define TMP_DNODE_DIR TD_TMP_DIR_PATH "dumpsdb" +#define TMP_MNODE_DIR TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" +#define TMP_SDB_DATA_DIR TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "data" +#define TMP_SDB_SYNC_DIR TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "sync" +#define TMP_SDB_MNODE_JSON TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "mnode.json" +#define TMP_SDB_DATA_FILE TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "data" TD_DIRSEP "sdb.data" +#define TMP_SDB_RAFT_CFG_FILE TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_config.json" +#define TMP_SDB_RAFT_STORE_FILE TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_store.json" + +void reportStartup(const char *name, const char *desc) {} + +void sendRsp(SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); } + +int32_t sendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { + terrno = TSDB_CODE_INVALID_PTR; + return -1; +} + +char *i642str(int64_t val) { + static char str[24] = {0}; + snprintf(str, sizeof(str), "%" PRId64, val); + return str; +} + +void dumpFunc(SSdb *pSdb, SJson *json) {} + +void dumpDb(SSdb *pSdb, SJson *json) { + void *pIter = NULL; + SJson *items = tjsonCreateObject(); + tjsonAddItemToObject(json, "dbs", items); + + while (1) { + SDbObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SJson *item = tjsonCreateObject(); + tjsonAddItemToObject(items, "db", item); + + tjsonAddStringToObject(item, "name", pObj->name); + tjsonAddStringToObject(item, "acct", pObj->acct); + tjsonAddStringToObject(item, "createUser", pObj->createUser); + tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); + tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); + tjsonAddStringToObject(item, "uid", i642str(pObj->uid)); + tjsonAddIntegerToObject(item, "cfgVersion", pObj->cfgVersion); + tjsonAddIntegerToObject(item, "vgVersion", pObj->vgVersion); + tjsonAddIntegerToObject(item, "numOfVgroups", pObj->cfg.numOfVgroups); + tjsonAddIntegerToObject(item, "numOfStables", pObj->cfg.numOfStables); + tjsonAddIntegerToObject(item, "buffer", pObj->cfg.buffer); + tjsonAddIntegerToObject(item, "pageSize", pObj->cfg.pageSize); + tjsonAddIntegerToObject(item, "pages", pObj->cfg.pages); + tjsonAddIntegerToObject(item, "cacheLastSize", pObj->cfg.cacheLastSize); + tjsonAddIntegerToObject(item, "daysPerFile", pObj->cfg.daysPerFile); + tjsonAddIntegerToObject(item, "daysToKeep0", pObj->cfg.daysToKeep0); + tjsonAddIntegerToObject(item, "daysToKeep1", pObj->cfg.daysToKeep1); + tjsonAddIntegerToObject(item, "daysToKeep2", pObj->cfg.daysToKeep2); + tjsonAddIntegerToObject(item, "minRows", pObj->cfg.minRows); + tjsonAddIntegerToObject(item, "maxRows", pObj->cfg.maxRows); + tjsonAddIntegerToObject(item, "precision", pObj->cfg.precision); + tjsonAddIntegerToObject(item, "compression", pObj->cfg.compression); + tjsonAddIntegerToObject(item, "replications", pObj->cfg.replications); + tjsonAddIntegerToObject(item, "strict", pObj->cfg.strict); + tjsonAddIntegerToObject(item, "cacheLast", pObj->cfg.cacheLast); + tjsonAddIntegerToObject(item, "hashMethod", pObj->cfg.hashMethod); + tjsonAddIntegerToObject(item, "numOfRetensions", pObj->cfg.numOfRetensions); + tjsonAddIntegerToObject(item, "schemaless", pObj->cfg.schemaless); + tjsonAddIntegerToObject(item, "walLevel", pObj->cfg.walLevel); + tjsonAddIntegerToObject(item, "walFsyncPeriod", pObj->cfg.walFsyncPeriod); + tjsonAddIntegerToObject(item, "walRetentionPeriod", pObj->cfg.walRetentionPeriod); + tjsonAddIntegerToObject(item, "walRetentionSize", pObj->cfg.walRetentionSize); + tjsonAddIntegerToObject(item, "walRollPeriod", pObj->cfg.walRollPeriod); + tjsonAddIntegerToObject(item, "walSegmentSize", pObj->cfg.walSegmentSize); + + sdbRelease(pSdb, pObj); + } +} + +void dumpStb(SSdb *pSdb, SJson *json) { + void *pIter = NULL; + SJson *items = tjsonCreateObject(); + tjsonAddItemToObject(json, "stbs", items); + + while (1) { + SStbObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SJson *item = tjsonCreateObject(); + tjsonAddItemToObject(items, "stb", item); + + tjsonAddStringToObject(item, "name", pObj->name); + tjsonAddStringToObject(item, "db", pObj->db); + tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); + tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); + tjsonAddStringToObject(item, "uid", i642str(pObj->uid)); + tjsonAddStringToObject(item, "dbUid", i642str(pObj->dbUid)); + tjsonAddIntegerToObject(item, "tagVer", pObj->tagVer); + tjsonAddIntegerToObject(item, "colVer", pObj->colVer); + tjsonAddIntegerToObject(item, "nextColId", pObj->nextColId); + tjsonAddIntegerToObject(item, "watermark1", pObj->watermark[0]); + tjsonAddIntegerToObject(item, "watermark2", pObj->watermark[1]); + tjsonAddIntegerToObject(item, "maxdelay1", pObj->maxdelay[0]); + tjsonAddIntegerToObject(item, "maxdelay2", pObj->maxdelay[1]); + tjsonAddIntegerToObject(item, "ttl", pObj->ttl); + tjsonAddIntegerToObject(item, "numOfColumns", pObj->numOfColumns); + tjsonAddIntegerToObject(item, "numOfTags", pObj->numOfTags); + tjsonAddIntegerToObject(item, "commentLen", pObj->commentLen); + tjsonAddIntegerToObject(item, "ast1Len", pObj->ast1Len); + tjsonAddIntegerToObject(item, "ast2Len", pObj->ast2Len); + + sdbRelease(pSdb, pObj); + } +} + +void dumpSma(SSdb *pSdb, SJson *json) {} + +void dumpVgroup(SSdb *pSdb, SJson *json) {} + +void dumpTopic(SSdb *pSdb, SJson *json) {} + +void dumpConsumber(SSdb *pSdb, SJson *json) {} + +void dumpSubscribe(SSdb *pSdb, SJson *json) {} + +void dumpOffset(SSdb *pSdb, SJson *json) {} + +void dumpStream(SSdb *pSdb, SJson *json) {} + +void dumpAcct(SSdb *pSdb, SJson *json) { + void *pIter = NULL; + SJson *items = tjsonCreateObject(); + tjsonAddItemToObject(json, "accts", items); + + while (1) { + SAcctObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_ACCT, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SJson *item = tjsonCreateObject(); + tjsonAddItemToObject(items, "acct", item); + + tjsonAddStringToObject(item, "acct", pObj->acct); + tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); + tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); + tjsonAddIntegerToObject(item, "acctId", pObj->acctId); + + sdbRelease(pSdb, pObj); + } +} + +void dumpAuth(SSdb *pSdb, SJson *json) {} + +void dumpUser(SSdb *pSdb, SJson *json) { + void *pIter = NULL; + SJson *items = tjsonCreateObject(); + tjsonAddItemToObject(json, "users", items); + + while (1) { + SUserObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SJson *item = tjsonCreateObject(); + tjsonAddItemToObject(items, "user", item); + + tjsonAddStringToObject(item, "name", pObj->user); + tjsonAddStringToObject(item, "pass", pObj->pass); + tjsonAddStringToObject(item, "acct", pObj->acct); + tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); + tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); + tjsonAddIntegerToObject(item, "superUser", pObj->superUser); + tjsonAddIntegerToObject(item, "authVersion", pObj->authVersion); + tjsonAddIntegerToObject(item, "numOfReadDbs", taosHashGetSize(pObj->readDbs)); + tjsonAddIntegerToObject(item, "numOfWriteDbs", taosHashGetSize(pObj->writeDbs)); + + sdbRelease(pSdb, pObj); + } +} + +void dumpDnode(SSdb *pSdb, SJson *json) { + void *pIter = NULL; + SJson *items = tjsonCreateObject(); + tjsonAddItemToObject(json, "dnodes", items); + + while (1) { + SDnodeObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SJson *item = tjsonCreateObject(); + tjsonAddItemToObject(items, "dnode", item); + + tjsonAddIntegerToObject(item, "id", pObj->id); + tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); + tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); + tjsonAddIntegerToObject(item, "port", pObj->port); + tjsonAddStringToObject(item, "fqdn", pObj->fqdn); + + sdbRelease(pSdb, pObj); + } +} + +void dumpBnode(SSdb *pSdb, SJson *json) {} + +void dumpSnode(SSdb *pSdb, SJson *json) {} + +void dumpQnode(SSdb *pSdb, SJson *json) {} + +void dumpMnode(SSdb *pSdb, SJson *json) { + void *pIter = NULL; + SJson *items = tjsonCreateObject(); + tjsonAddItemToObject(json, "mnodes", items); + + while (1) { + SMnodeObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SJson *item = tjsonCreateObject(); + tjsonAddItemToObject(items, "mnode", item); + + tjsonAddIntegerToObject(item, "id", pObj->id); + tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); + tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); + + sdbRelease(pSdb, pObj); + } +} + +void dumpCluster(SSdb *pSdb, SJson *json) { + void *pIter = NULL; + SJson *items = tjsonCreateObject(); + tjsonAddItemToObject(json, "clusters", items); + + while (1) { + SClusterObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_CLUSTER, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SJson *item = tjsonCreateObject(); + tjsonAddItemToObject(items, "cluster", item); + + tjsonAddStringToObject(item, "id", i642str(pObj->id)); + tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); + tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); + tjsonAddStringToObject(item, "name", pObj->name); + + sdbRelease(pSdb, pObj); + } +} + +void dumpTrans(SSdb *pSdb, SJson *json) { + void *pIter = NULL; + SJson *items = tjsonCreateObject(); + tjsonAddItemToObject(json, "transactions", items); + + while (1) { + STrans *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_TRANS, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SJson *item = tjsonCreateObject(); + tjsonAddItemToObject(items, "trans", item); + + tjsonAddIntegerToObject(item, "id", pObj->id); + tjsonAddIntegerToObject(item, "stage", pObj->stage); + tjsonAddIntegerToObject(item, "policy", pObj->policy); + tjsonAddIntegerToObject(item, "conflict", pObj->conflict); + tjsonAddIntegerToObject(item, "exec", pObj->exec); + tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); + tjsonAddStringToObject(item, "dbname1", pObj->dbname1); + tjsonAddStringToObject(item, "dbname2", pObj->dbname2); + tjsonAddIntegerToObject(item, "commitLogNum", taosArrayGetSize(pObj->commitActions)); + tjsonAddIntegerToObject(item, "redoActionNum", taosArrayGetSize(pObj->redoActions)); + tjsonAddIntegerToObject(item, "undoActionNum", taosArrayGetSize(pObj->undoActions)); + + sdbRelease(pSdb, pObj); + } +} + +void dumpHeader(SSdb *pSdb, SJson *json) { + tjsonAddIntegerToObject(json, "sver", 1); + tjsonAddStringToObject(json, "applyIndex", i642str(pSdb->applyIndex)); + tjsonAddStringToObject(json, "applyTerm", i642str(pSdb->applyTerm)); + tjsonAddStringToObject(json, "applyConfig", i642str(pSdb->applyConfig)); + + SJson *maxIdsJson = tjsonCreateObject(); + tjsonAddItemToObject(json, "maxIds", maxIdsJson); + for (int32_t i = 0; i < SDB_MAX; ++i) { + int64_t maxId = 0; + if (i < SDB_MAX) { + maxId = pSdb->maxId[i]; + } + tjsonAddStringToObject(maxIdsJson, sdbTableName(i), i642str(maxId)); + } + + SJson *tableVersJson = tjsonCreateObject(); + tjsonAddItemToObject(json, "tableVers", tableVersJson); + for (int32_t i = 0; i < SDB_MAX; ++i) { + int64_t tableVer = 0; + if (i < SDB_MAX) { + tableVer = pSdb->tableVer[i]; + } + tjsonAddStringToObject(tableVersJson, sdbTableName(i), i642str(tableVer)); + } +} + +int32_t dumpSdb() { + wDebugFlag = 0; + mDebugFlag = 0; + sDebugFlag = 0; + + SMsgCb msgCb = {0}; + msgCb.reportStartupFp = reportStartup; + msgCb.sendReqFp = sendReq; + msgCb.sendRspFp = sendRsp; + msgCb.mgmt = (SMgmtWrapper *)(&msgCb); // hack + tmsgSetDefault(&msgCb); + walInit(); + syncInit(); + + SMnodeOpt opt = {.msgCb = msgCb}; + SMnode *pMnode = mndOpen(TMP_MNODE_DIR, &opt); + if (pMnode == NULL) return -1; + + SSdb *pSdb = pMnode->pSdb; + SJson *json = tjsonCreateObject(); + dumpHeader(pSdb, json); + dumpFunc(pSdb, json); + dumpDb(pSdb, json); + dumpStb(pSdb, json); + dumpSma(pSdb, json); + dumpVgroup(pSdb, json); + dumpTopic(pSdb, json); + dumpConsumber(pSdb, json); + dumpSubscribe(pSdb, json); + dumpOffset(pSdb, json); + dumpStream(pSdb, json); + dumpAcct(pSdb, json); + dumpAuth(pSdb, json); + dumpUser(pSdb, json); + dumpDnode(pSdb, json); + dumpBnode(pSdb, json); + dumpSnode(pSdb, json); + dumpQnode(pSdb, json); + dumpMnode(pSdb, json); + dumpCluster(pSdb, json); + dumpTrans(pSdb, json); + + char *pCont = tjsonToString(json); + int32_t contLen = strlen(pCont); + char file[] = "sdb.json"; + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to write %s since %s", file, terrstr()); + return -1; + } + taosWriteFile(pFile, pCont, contLen); + taosWriteFile(pFile, "\n", 1); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + tjsonDelete(json); + taosMemoryFree(pCont); + taosRemoveDir(TMP_DNODE_DIR); + return 0; +} + +int32_t parseArgs(int32_t argc, char *argv[]) { + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0) { + if (i < argc - 1) { + if (strlen(argv[++i]) >= PATH_MAX) { + printf("config file path overflow"); + return -1; + } + tstrncpy(configDir, argv[i], PATH_MAX); + } else { + printf("'-c' requires a parameter, default is %s\n", configDir); + return -1; + } + } else { + printf("-c Configuration directory. \n"); + return -1; + } + } + + if (taosCreateLog("dumplog", 1, configDir, NULL, NULL, NULL, NULL, 1) != 0) { + printf("failed to dump since init log error\n"); + return -1; + } + + if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) { + printf("failed to dump since read config error\n"); + return -1; + } + + char mnodeJson[PATH_MAX] = {0}; + char dataFile[PATH_MAX] = {0}; + char raftCfgFile[PATH_MAX] = {0}; + char raftStoreFile[PATH_MAX] = {0}; + snprintf(mnodeJson, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "mnode.json", tsDataDir); + snprintf(dataFile, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "data" TD_DIRSEP "sdb.data", tsDataDir); + snprintf(raftCfgFile, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_config.json", tsDataDir); + snprintf(raftStoreFile, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_store.json", tsDataDir); + + char cmd[PATH_MAX * 2] = {0}; + snprintf(cmd, sizeof(cmd), "rm -rf %s", TMP_DNODE_DIR); + + system(cmd); +#ifdef WINDOWS + taosMulMkDir(TMP_SDB_DATA_DIR); + taosMulMkDir(TMP_SDB_SYNC_DIR); + snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", mnodeJson, TMP_SDB_MNODE_JSON); + system(cmd); + snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", dataFile, TMP_SDB_DATA_FILE); + system(cmd); + snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", raftCfgFile, TMP_SDB_RAFT_CFG_FILE); + system(cmd); + snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", raftStoreFile, TMP_SDB_RAFT_STORE_FILE); + system(cmd); +#else + snprintf(cmd, sizeof(cmd), "mkdir -p %s", TMP_SDB_DATA_DIR); + system(cmd); + snprintf(cmd, sizeof(cmd), "mkdir -p %s", TMP_SDB_SYNC_DIR); + system(cmd); + snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", mnodeJson, TMP_SDB_MNODE_JSON); + system(cmd); + snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", dataFile, TMP_SDB_DATA_FILE); + system(cmd); + snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", raftCfgFile, TMP_SDB_RAFT_CFG_FILE); + system(cmd); + snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", raftStoreFile, TMP_SDB_RAFT_STORE_FILE); + system(cmd); +#endif + + strcpy(tsDataDir, TMP_DNODE_DIR); + return 0; +} + +int32_t main(int32_t argc, char *argv[]) { + if (parseArgs(argc, argv) != 0) { + return -1; + } + + return dumpSdb(); +} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c new file mode 100644 index 0000000000..1fd1def263 --- /dev/null +++ b/utils/test/c/sml_test.c @@ -0,0 +1,1191 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "taos.h" +#include "types.h" + +int smlProcess_influx_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 " + "load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=" + "124,velocity=0,heading=221,grade=0 1451606401000000000", + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 " + "load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=" + "124,velocity=0,heading=221,grade=0,fuel_consumption=25 1451607402000000000", + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 " + "load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=" + "124,heading=221,grade=0,fuel_consumption=25 1451608403000000000", + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 " + "fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0," + "heading=221,grade=0,fuel_consumption=25 1451609404000000000", + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 fuel_consumption=25,grade=0 " + "1451619405000000000", + "readings,name=truck_1,fleet=South,driver=Albert,model=F-150,device_version=v1.5 " + "load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=72.45258,longitude=68.83761,elevation=" + "255,velocity=0,heading=181,grade=0,fuel_consumption=25 1451606406000000000", + "readings,name=truck_2,driver=Derek,model=F-150,device_version=v1.5 " + "load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=" + "428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451606407000000000", + "readings,name=truck_2,fleet=North,driver=Derek,model=F-150 " + "load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=" + "428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451609408000000000", + "readings,fleet=South,name=truck_0,driver=Trish,model=H-2,device_version=v2.3 fuel_consumption=25,grade=0 " + "1451629409000000000", + "stable,t1=t1,t2=t2,t3=t3 c1=1,c2=2,c3=\"kk\",c4=4 1451629501000000000", + "stable,t2=t2,t1=t1,t3=t3 c1=1,c3=\"\",c4=4 1451629602000000000", + }; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + + return code; +} + +int smlProcess_telnet_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = {"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", + "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ", + "sys.if.bytes.out 1479496102 1.3E3 network=tcp", + " sys.procs.running 1479496100 42 host=web01 "}; + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int smlProcess_json1_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "[" + " {" + " \"metric\": \"sys.cpu.nice\"," + " \"timestamp\": 0," + " \"value\": 18," + " \"tags\": {" + " \"host\": \"web01\"," + " \"id\": \"t1\"," + " \"dc\": \"lga\"" + " }" + " }," + " {" + " \"metric\": \"sys.cpu.nice\"," + " \"timestamp\": 1346846400," + " \"value\": 9," + " \"tags\": {" + " \"host\": \"web02\"," + " \"dc\": \"lga\"" + " }" + " }" + "]"}; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int smlProcess_json2_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "{" + " \"metric\": \"meter_current0\"," + " \"timestamp\": {" + " \"value\" : 1346846400," + " \"type\" : \"s\"" + " }," + " \"value\": {" + " \"value\" : 10.3," + " \"type\" : \"i64\"" + " }," + " \"tags\": {" + " \"groupid\": { " + " \"value\" : 2," + " \"type\" : \"bigint\"" + " }," + " \"location\": { " + " \"value\" : \"北京\"," + " \"type\" : \"binary\"" + " }," + " \"id\": \"d1001\"" + " }" + "}"}; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int smlProcess_json3_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "{" + " \"metric\": \"meter_current1\"," + " \"timestamp\": {" + " \"value\" : 1346846400," + " \"type\" : \"s\"" + " }," + " \"value\": {" + " \"value\" : 10.3," + " \"type\" : \"i64\"" + " }," + " \"tags\": {" + " \"t1\": { " + " \"value\" : 2," + " \"type\" : \"bigint\"" + " }," + " \"t2\": { " + " \"value\" : 2," + " \"type\" : \"int\"" + " }," + " \"t3\": { " + " \"value\" : 2," + " \"type\" : \"i16\"" + " }," + " \"t4\": { " + " \"value\" : 2," + " \"type\" : \"i8\"" + " }," + " \"t5\": { " + " \"value\" : 2," + " \"type\" : \"f32\"" + " }," + " \"t6\": { " + " \"value\" : 2," + " \"type\" : \"double\"" + " }," + " \"t7\": { " + " \"value\" : \"8323\"," + " \"type\" : \"binary\"" + " }," + " \"t8\": { " + " \"value\" : \"北京\"," + " \"type\" : \"nchar\"" + " }," + " \"t9\": { " + " \"value\" : true," + " \"type\" : \"bool\"" + " }," + " \"id\": \"d1001\"" + " }" + "}"}; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int smlProcess_json4_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "{" + " \"metric\": \"meter_current2\"," + " \"timestamp\": {" + " \"value\" : 1346846500000," + " \"type\" : \"ms\"" + " }," + " \"value\": \"ni\"," + " \"tags\": {" + " \"t1\": { " + " \"value\" : 20," + " \"type\" : \"i64\"" + " }," + " \"t2\": { " + " \"value\" : 25," + " \"type\" : \"i32\"" + " }," + " \"t3\": { " + " \"value\" : 2," + " \"type\" : \"smallint\"" + " }," + " \"t4\": { " + " \"value\" : 2," + " \"type\" : \"tinyint\"" + " }," + " \"t5\": { " + " \"value\" : 2," + " \"type\" : \"float\"" + " }," + " \"t6\": { " + " \"value\" : 0.2," + " \"type\" : \"f64\"" + " }," + " \"t7\": \"nsj\"," + " \"t8\": { " + " \"value\" : \"北京\"," + " \"type\" : \"nchar\"" + " }," + " \"t9\": false," + " \"id\": \"d1001\"" + " }" + "}"}; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int sml_TD15662_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db precision 'ns' schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "hetrey c0=f,c1=127i8 1626006833639", + "hetrey,t1=r c0=f,c1=127i8 1626006833640", + }; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int sml_TD15742_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "test_ms,t0=t c0=f 1626006833641", + }; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int sml_16384_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + const char *sql[] = { + "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=t,c1=127i8 1626006833639000000", + }; + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, 1, TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + if(code) return code; + + const char *sql1[] = { + "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=f,c1=127i8,c11=L\"ncharColValue\",c10=t 1626006833639000000", + }; + pRes = taos_schemaless_insert(taos, (char **)sql1, 1, TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int sml_oom_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + const char *sql[] = { + //"test_ms,t0=t c0=f 1626006833641", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"pgxbrbga\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"gviggpmi\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"cexkarjn\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"rzwwuoxu\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"xphrlkey\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"llsawebj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"jwpkipff\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"euzzhcvu\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"jumhnsvw\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"fnetgdhj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"vrmmpgqe\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"lnpfjapr\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"gvbhmsfr\",t8=L\"ncharTagValue\" " + "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"kydxrxwc\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"pfyarryq\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"uxptotap\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"prolhudh\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ttxaxnac\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"dfgvmjmz\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"bloextkn\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"dvjxwzsi\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"aigjomaf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"refbidtf\",t8=L\"ncharTagValue\" " + "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"vuanlfpz\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"nbpajxkx\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ktzzauxh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"prcwdjct\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"vmbhvjtp\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"liuddtuz\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"pddsktow\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"algldlvl\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"mlmnjgdl\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"oiynpcog\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"wmynbagb\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"asvyulrm\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ohaacrkp\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"ytyejhiq\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"bbznuerb\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"lpebcibw\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"xmqrbafv\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"lnmwpdne\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"jpcsjqun\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"mmxqmavz\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"hhsbgaow\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"uwogyuud\",t8=L\"ncharTagValue\" " + "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ytxpaxnk\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"wouwdvtt\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"iitwikkh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"lgyzuyaq\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"bdtiigxi\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"qpnsvdhw\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"pjxihgvu\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"ksxkfetn\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ocukufqs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"qzerxmpe\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"qwcfdyxs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"jldrpmmd\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"lucxlfzc\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"rcewrvya\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"dknvaphs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"nxtxgzdr\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"mbvuugwz\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"uikakffu\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"mwmtqsma\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"bfcxrrpa\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ksajygdj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"vmhhszyv\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"urwjgvut\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"jrvytcxy\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"evqkzygh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"zitdznhg\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"tpqekrxa\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"yrrbgjtk\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"bnphiuyq\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"huknehjn\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"iudbxfke\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"fjmolwbn\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"gukzgcjs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"bjvdtlgq\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"phxnesxh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"qgpgckvc\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"yechqtfa\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"pbouxywy\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"kxtuojyo\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"txaniwlj\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"fixgufrj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"okzvalwq\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"iitawgbn\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"gayvmird\",t8=L\"ncharTagValue\" " + "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"dprkfjph\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"kmuccshq\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"vkslsdsd\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"dukccdqk\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"leztxmqf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"kltixbwz\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"xqhkweef\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"idxsimvz\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"vbruvcpk\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"uxandqkd\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"dsiosysh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"kxuyanpp\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"wkrktags\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"yvizzpiv\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ddnefben\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"novmfmbc\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"fnusxsfu\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"ouerfjap\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"sigognkf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"slvzhede\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"bknerect\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"tmhcdfjb\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"hpnoanpp\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"okmhelnc\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"xcernjin\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"jdmiismg\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"tmnqozrf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"zgwrftkx\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"zyamlwwh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"nuedqcro\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"lpsvyqaa\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"mneitsul\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"vpleinwb\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"njxuaedy\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"sdgxpqmu\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"yjirrebp\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ikqndzfj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"ghnfdxhr\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"hrwczpvo\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"nattumpb\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"zoyfzazn\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"rdwemofy\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"phkgsjeg\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"pyhvvjrt\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"zfslyton\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"bxwjzeri\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"uovzzgjv\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"cfjmacvr\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"jefqgzqx\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"njrksxmr\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"mhvabvgn\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"kfekjltr\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"lexfaaby\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"zbblsmwq\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"oqcombkx\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"rcdmhzyw\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"otksuean\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"itbdvowq\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"tswtmhex\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"xoukkzid\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"guangmpq\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"rayxzuky\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"lspwucrv\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"pdprzzkf\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"sddqrtza\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"kabndgkx\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"aglnqqxs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"fiwpzmdr\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"hxctooen\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"pckjpwyh\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ivmvsbai\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"eljdclst\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"rwgdctie\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"zlnthxoz\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ljtxelle\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"llfggdpy\",t8=L\"ncharTagValue\" " + "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"tvnridze\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"hxjpgube\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"zmldmquq\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"bggqwcoj\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"drksfofm\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"jcsixens\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"cdwnwhaf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"nngpumuq\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"hylgooci\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"cozeyjys\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"lcgpfcsa\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"qdtzhtyd\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"txpubynb\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"gbslzbtu\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"buihcpcl\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"ayqezaiq\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"zgkgtilj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"bcjopqif\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"mfzxiaqt\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"xmnlqxoj\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"reyiklyf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"xssuomhk\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"liazkjll\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"nigjlblo\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"vmojyznk\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"dotkbvrz\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"kuwdyydw\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"slsfqydw\",t8=L\"ncharTagValue\" " + "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"zyironhd\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"pktwfhzi\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"xybavsvh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"pyrxemvx\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"tlfihwjs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"neumakmg\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"wxqingoa\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + }; + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int sml_16368_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "[{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639000, \"type\": \"us\"}, \"value\": 1, " + "\"tags\": {\"t1\": 3, \"t2\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t3\", \"type\": " + "\"binary\"}}}," + "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833739000, \"type\": \"us\"}, \"value\": 2, " + "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " + "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," + "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639100, \"type\": \"us\"}, \"value\": 3, " + "\"tags\": {\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t3\": {\"value\": \"ste\", \"type\": \"nchar\"}}}," + "{\"metric\": \"stf567890\", \"timestamp\": {\"value\": 1626006833639200, \"type\": \"us\"}, \"value\": 4, " + "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"bigint\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " + "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," + "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639300, \"type\": \"us\"}, \"value\": " + "{\"value\": 5, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t2\": 5.0, " + "\"t3\": {\"value\": \"t4\", \"type\": \"binary\"}}}," + "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639400, \"type\": \"us\"}, \"value\": " + "{\"value\": 6, \"type\": \"double\"}, \"tags\": {\"t2\": 5.0, \"t3\": {\"value\": \"ste2\", \"type\": " + "\"nchar\"}}}," + "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006834639400, \"type\": \"us\"}, \"value\": " + "{\"value\": 7, \"type\": \"double\"}, \"tags\": {\"t2\": {\"value\": 5.0, \"type\": \"double\"}, \"t3\": " + "{\"value\": \"ste2\", \"type\": \"nchar\"}}}," + "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833839006, \"type\": \"us\"}, \"value\": " + "{\"value\": 8, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": " + "{\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, " + "\"type\": \"double\"}}}," + "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833939007, \"type\": \"us\"}, \"value\": " + "{\"value\": 9, \"type\": \"double\"}, \"tags\": {\"t1\": 4, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " + "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}]"}; + pRes = taos_schemaless_insert(taos, (char **)sql, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int sml_dup_time_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + const char *sql[] = {//"test_ms,t0=t c0=f 1626006833641", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=false,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006834639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006835639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006836639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006837639000000"}; + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int sml_16960_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "[" + "{" + "\"timestamp\":" + "" + "{ \"value\": 1349020800000, \"type\": \"ms\" }" + "," + "\"value\":" + "" + "{ \"value\": 830525384, \"type\": \"int\" }" + "," + "\"tags\": {" + "\"id\": \"stb00_0\"," + "\"t0\":" + "" + "{ \"value\": 83972721, \"type\": \"int\" }" + "," + "\"t1\":" + "" + "{ \"value\": 539147525, \"type\": \"int\" }" + "," + "\"t2\":" + "" + "{ \"value\": 618258572, \"type\": \"int\" }" + "," + "\"t3\":" + "" + "{ \"value\": -10536201, \"type\": \"int\" }" + "," + "\"t4\":" + "" + "{ \"value\": 349227409, \"type\": \"int\" }" + "," + "\"t5\":" + "" + "{ \"value\": 249347042, \"type\": \"int\" }" + "}," + "\"metric\": \"stb0\"" + "}," + "{" + "\"timestamp\":" + "" + "{ \"value\": 1349020800001, \"type\": \"ms\" }" + "," + "\"value\":" + "" + "{ \"value\": -588348364, \"type\": \"int\" }" + "," + "\"tags\": {" + "\"id\": \"stb00_0\"," + "\"t0\":" + "" + "{ \"value\": 83972721, \"type\": \"int\" }" + "," + "\"t1\":" + "" + "{ \"value\": 539147525, \"type\": \"int\" }" + "," + "\"t2\":" + "" + "{ \"value\": 618258572, \"type\": \"int\" }" + "," + "\"t3\":" + "" + "{ \"value\": -10536201, \"type\": \"int\" }" + "," + "\"t4\":" + "" + "{ \"value\": 349227409, \"type\": \"int\" }" + "," + "\"t5\":" + "" + "{ \"value\": 249347042, \"type\": \"int\" }" + "}," + "\"metric\": \"stb0\"" + "}," + "{" + "\"timestamp\":" + "" + "{ \"value\": 1349020800002, \"type\": \"ms\" }" + "," + "\"value\":" + "" + "{ \"value\": -370310823, \"type\": \"int\" }" + "," + "\"tags\": {" + "\"id\": \"stb00_0\"," + "\"t0\":" + "" + "{ \"value\": 83972721, \"type\": \"int\" }" + "," + "\"t1\":" + "" + "{ \"value\": 539147525, \"type\": \"int\" }" + "," + "\"t2\":" + "" + "{ \"value\": 618258572, \"type\": \"int\" }" + "," + "\"t3\":" + "" + "{ \"value\": -10536201, \"type\": \"int\" }" + "," + "\"t4\":" + "" + "{ \"value\": 349227409, \"type\": \"int\" }" + "," + "\"t5\":" + "" + "{ \"value\": 249347042, \"type\": \"int\" }" + "}," + "\"metric\": \"stb0\"" + "}," + "{" + "\"timestamp\":" + "" + "{ \"value\": 1349020800003, \"type\": \"ms\" }" + "," + "\"value\":" + "" + "{ \"value\": -811250191, \"type\": \"int\" }" + "," + "\"tags\": {" + "\"id\": \"stb00_0\"," + "\"t0\":" + "" + "{ \"value\": 83972721, \"type\": \"int\" }" + "," + "\"t1\":" + "" + "{ \"value\": 539147525, \"type\": \"int\" }" + "," + "\"t2\":" + "" + "{ \"value\": 618258572, \"type\": \"int\" }" + "," + "\"t3\":" + "" + "{ \"value\": -10536201, \"type\": \"int\" }" + "," + "\"t4\":" + "" + "{ \"value\": 349227409, \"type\": \"int\" }" + "," + "\"t5\":" + "" + "{ \"value\": 249347042, \"type\": \"int\" }" + "}," + "\"metric\": \"stb0\"" + "}," + "{" + "\"timestamp\":" + "" + "{ \"value\": 1349020800004, \"type\": \"ms\" }" + "," + "\"value\":" + "" + "{ \"value\": -330340558, \"type\": \"int\" }" + "," + "\"tags\": {" + "\"id\": \"stb00_0\"," + "\"t0\":" + "" + "{ \"value\": 83972721, \"type\": \"int\" }" + "," + "\"t1\":" + "" + "{ \"value\": 539147525, \"type\": \"int\" }" + "," + "\"t2\":" + "" + "{ \"value\": 618258572, \"type\": \"int\" }" + "," + "\"t3\":" + "" + "{ \"value\": -10536201, \"type\": \"int\" }" + "," + "\"t4\":" + "" + "{ \"value\": 349227409, \"type\": \"int\" }" + "," + "\"t5\":" + "" + "{ \"value\": 249347042, \"type\": \"int\" }" + "}," + "\"metric\": \"stb0\"" + "}" + "]"}; + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int sml_add_tag_col_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + const char *sql[] = { + "macylr,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000" + }; + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + if (code) return code; + + const char *sql1[] = { + "macylr,id=macylr_17875_1804,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\",t11=127i8,t10=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c8=L\"ncharColValue\",c9=7u64,c11=L\"ncharColValue\",c10=f 1626006833639000000" + }; + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + taos_free_result(pRes); + + return code; +} + +int smlProcess_18784_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "disk,device=sdc inodes_used=176059i,total=1081101176832i 1661943960000000000", + "disk,device=sdc inodes_free=66932805i 1661943960000000000", + }; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); + int code = taos_errno(pRes); + ASSERT(!code); + ASSERT(taos_affected_rows(pRes) == 2); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from disk"); + ASSERT(pRes); + int fieldNum = taos_field_count(pRes); + ASSERT(fieldNum == 5); + printf("fieldNum:%d\n", fieldNum); + TAOS_ROW row = NULL; + int32_t rowIndex = 0; + while((row = taos_fetch_row(pRes)) != NULL) { + int64_t ts = *(int64_t*)row[0]; + int64_t used = *(int64_t*)row[1]; + int64_t total = *(int64_t*)row[2]; + int64_t freed = *(int64_t*)row[3]; + if(rowIndex == 0){ + ASSERT(ts == 1661943960000); + ASSERT(used == 176059); + ASSERT(total == 1081101176832); + ASSERT(freed == 66932805); +// ASSERT_EQ(latitude, 24.5208); +// ASSERT_EQ(longitude, 28.09377); +// ASSERT_EQ(elevation, 428); +// ASSERT_EQ(velocity, 0); +// ASSERT_EQ(heading, 304); +// ASSERT_EQ(grade, 0); +// ASSERT_EQ(fuel_consumption, 25); + }else{ +// ASSERT(0); + } + rowIndex++; + } + taos_free_result(pRes); + + return code; +} + +int main(int argc, char *argv[]) { + int ret = 0; + ret = smlProcess_influx_Test(); + ASSERT(!ret); + ret = smlProcess_telnet_Test(); + ASSERT(!ret); + ret = smlProcess_json1_Test(); + ASSERT(!ret); + ret = smlProcess_json2_Test(); + ASSERT(!ret); + ret = smlProcess_json3_Test(); + ASSERT(!ret); + ret = smlProcess_json4_Test(); + ASSERT(!ret); + ret = sml_TD15662_Test(); + ASSERT(!ret); + ret = sml_TD15742_Test(); + ASSERT(!ret); + ret = sml_16384_Test(); + ASSERT(!ret); + ret = sml_oom_Test(); + ASSERT(!ret); + ret = sml_16368_Test(); + ASSERT(!ret); + ret = sml_dup_time_Test(); + ASSERT(!ret); + ret = sml_16960_Test(); + ASSERT(!ret); + ret = sml_add_tag_col_Test(); + ASSERT(!ret); + ret = smlProcess_18784_Test(); + ASSERT(!ret); + return ret; +} diff --git a/utils/test/c/tmqDemo.c b/utils/test/c/tmqDemo.c new file mode 100644 index 0000000000..784b45c92b --- /dev/null +++ b/utils/test/c/tmqDemo.c @@ -0,0 +1,705 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include +#include +// #include + +#include "taos.h" +#include "taoserror.h" +#include "tlog.h" + +#define GREEN "\033[1;32m" +#define NC "\033[0m" +#define min(a, b) (((a) < (b)) ? (a) : (b)) + +#define MAX_SQL_STR_LEN (1024 * 1024) +#define MAX_ROW_STR_LEN (16 * 1024) + +enum _RUN_MODE { + TMQ_RUN_INSERT_AND_CONSUME, + TMQ_RUN_ONLY_INSERT, + TMQ_RUN_ONLY_CONSUME, + TMQ_RUN_MODE_BUTT, +}; + +typedef struct { + char dbName[32]; + char stbName[64]; + char resultFileName[256]; + char vnodeWalPath[256]; + int32_t numOfThreads; + int32_t numOfTables; + int32_t numOfVgroups; + int32_t runMode; + int32_t numOfColumn; + double ratio; + int32_t batchNumOfRow; + int32_t totalRowsOfPerTbl; + int64_t startTimestamp; + int32_t showMsgFlag; + int32_t simCase; + + int32_t totalRowsOfT2; +} SConfInfo; + +static SConfInfo g_stConfInfo = { + "tmqdb", + "stb", + "./tmqResult.txt", // output_file + "", // /data2/dnode/data/vnode/vnode2/wal", + 1, // threads + 1, // tables + 1, // vgroups + 0, // run mode + 1, // columns + 1, // ratio + 1, // batch size + 10000, // total rows for per table + 0, // 2020-01-01 00:00:00.000 + 0, // show consume msg switch + 0, // if run in sim case + 10000, +}; + +char* g_pRowValue = NULL; +TdFilePtr g_fp = NULL; + +static void printHelp() { + char indent[10] = " "; + printf("Used to test the performance while create table\n"); + + printf("%s%s\n", indent, "-c"); + printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); + printf("%s%s\n", indent, "-d"); + printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", g_stConfInfo.dbName); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", g_stConfInfo.stbName); + printf("%s%s\n", indent, "-f"); + printf("%s%s%s%s\n", indent, indent, "The file of result, default is ", g_stConfInfo.resultFileName); + printf("%s%s\n", indent, "-w"); + printf("%s%s%s%s\n", indent, indent, "The path of vnode of wal, default is ", g_stConfInfo.vnodeWalPath); + printf("%s%s\n", indent, "-t"); + printf("%s%s%s%d\n", indent, indent, "numOfThreads, default is ", g_stConfInfo.numOfThreads); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s%d\n", indent, indent, "numOfTables, default is ", g_stConfInfo.numOfTables); + printf("%s%s\n", indent, "-v"); + printf("%s%s%s%d\n", indent, indent, "numOfVgroups, default is ", g_stConfInfo.numOfVgroups); + printf("%s%s\n", indent, "-a"); + printf("%s%s%s%d\n", indent, indent, "runMode, default is ", g_stConfInfo.runMode); + printf("%s%s\n", indent, "-l"); + printf("%s%s%s%d\n", indent, indent, "numOfColumn, default is ", g_stConfInfo.numOfColumn); + printf("%s%s\n", indent, "-q"); + printf("%s%s%s%f\n", indent, indent, "ratio, default is ", g_stConfInfo.ratio); + printf("%s%s\n", indent, "-b"); + printf("%s%s%s%d\n", indent, indent, "batchNumOfRow, default is ", g_stConfInfo.batchNumOfRow); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%d\n", indent, indent, "totalRowsOfPerTbl, default is ", g_stConfInfo.totalRowsOfPerTbl); + printf("%s%s\n", indent, "-m"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "startTimestamp, default is ", g_stConfInfo.startTimestamp); + printf("%s%s\n", indent, "-g"); + printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag); + printf("%s%s\n", indent, "-sim"); + printf("%s%s%s%d\n", indent, indent, "simCase, default is ", g_stConfInfo.simCase); + + exit(EXIT_SUCCESS); +} + +void parseArgument(int32_t argc, char* argv[]) { + g_stConfInfo.startTimestamp = 1640966400000; // 2020-01-01 00:00:00.000 + + for (int32_t i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-d") == 0) { + strcpy(g_stConfInfo.dbName, argv[++i]); + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + strcpy(g_stConfInfo.stbName, argv[++i]); + } else if (strcmp(argv[i], "-w") == 0) { + strcpy(g_stConfInfo.vnodeWalPath, argv[++i]); + } else if (strcmp(argv[i], "-f") == 0) { + strcpy(g_stConfInfo.resultFileName, argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + g_stConfInfo.numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + g_stConfInfo.numOfTables = atoll(argv[++i]); + } else if (strcmp(argv[i], "-v") == 0) { + g_stConfInfo.numOfVgroups = atoi(argv[++i]); + } else if (strcmp(argv[i], "-a") == 0) { + g_stConfInfo.runMode = atoi(argv[++i]); + } else if (strcmp(argv[i], "-b") == 0) { + g_stConfInfo.batchNumOfRow = atoi(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + g_stConfInfo.totalRowsOfPerTbl = atoi(argv[++i]); + } else if (strcmp(argv[i], "-l") == 0) { + g_stConfInfo.numOfColumn = atoi(argv[++i]); + } else if (strcmp(argv[i], "-q") == 0) { + g_stConfInfo.ratio = atof(argv[++i]); + } else if (strcmp(argv[i], "-m") == 0) { + g_stConfInfo.startTimestamp = atol(argv[++i]); + } else if (strcmp(argv[i], "-g") == 0) { + g_stConfInfo.showMsgFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-sim") == 0) { + g_stConfInfo.simCase = atol(argv[++i]); + } else { + printf("%s unknow para: %s %s", GREEN, argv[++i], NC); + exit(-1); + } + } + + g_stConfInfo.totalRowsOfT2 = g_stConfInfo.totalRowsOfPerTbl * g_stConfInfo.ratio; + +#if 0 + pPrint("%s configDir:%s %s", GREEN, configDir, NC); + pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC); + pPrint("%s stbName:%s %s", GREEN, g_stConfInfo.stbName, NC); + pPrint("%s resultFileName:%s %s", GREEN, g_stConfInfo.resultFileName, NC); + pPrint("%s vnodeWalPath:%s %s", GREEN, g_stConfInfo.vnodeWalPath, NC); + pPrint("%s numOfTables:%d %s", GREEN, g_stConfInfo.numOfTables, NC); + pPrint("%s numOfThreads:%d %s", GREEN, g_stConfInfo.numOfThreads, NC); + pPrint("%s numOfVgroups:%d %s", GREEN, g_stConfInfo.numOfVgroups, NC); + pPrint("%s runMode:%d %s", GREEN, g_stConfInfo.runMode, NC); + pPrint("%s ratio:%f %s", GREEN, g_stConfInfo.ratio, NC); + pPrint("%s numOfColumn:%d %s", GREEN, g_stConfInfo.numOfColumn, NC); + pPrint("%s batchNumOfRow:%d %s", GREEN, g_stConfInfo.batchNumOfRow, NC); + pPrint("%s totalRowsOfPerTbl:%d %s", GREEN, g_stConfInfo.totalRowsOfPerTbl, NC); + pPrint("%s totalRowsOfT2:%d %s", GREEN, g_stConfInfo.totalRowsOfT2, NC); + pPrint("%s startTimestamp:%" PRId64" %s", GREEN, g_stConfInfo.startTimestamp, NC); + pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); +#endif +} + +static int running = 1; +/*static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }*/ + +// calc dir size (not include itself 4096Byte) +int64_t getDirectorySize(char* dir) { + TdDirPtr pDir; + TdDirEntryPtr pDirEntry; + int64_t totalSize = 0; + + if ((pDir = taosOpenDir(dir)) == NULL) { + fprintf(stderr, "Cannot open dir: %s\n", dir); + return -1; + } + + // lstat(dir, &statbuf); + // totalSize+=statbuf.st_size; + + while ((pDirEntry = taosReadDir(pDir)) != NULL) { + char subdir[1024]; + char* fileName = taosGetDirEntryName(pDirEntry); + sprintf(subdir, "%s/%s", dir, fileName); + + // printf("===d_name: %s\n", entry->d_name); + if (taosIsDir(subdir)) { + if (strcmp(".", fileName) == 0 || strcmp("..", fileName) == 0) { + continue; + } + + int64_t subDirSize = getDirectorySize(subdir); + totalSize += subDirSize; + } else if (0 == strcmp(strchr(fileName, '.'), ".log")) { // only calc .log file size, and not include .idx file + int64_t file_size = 0; + taosStatFile(subdir, &file_size, NULL); + totalSize += file_size; + } + } + + taosCloseDir(&pDir); + return totalSize; +} + +int queryDB(TAOS* taos, char* command) { + TAOS_RES* pRes = taos_query(taos, command); + int code = taos_errno(pRes); + if (code != 0) { + pError("failed to reason:%s, sql: %s", tstrerror(code), command); + taos_free_result(pRes); + return -1; + } + taos_free_result(pRes); + return 0; +} + +int32_t init_env() { + char sqlStr[1024] = {0}; + + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + sprintf(sqlStr, "create database if not exists %s vgroups %d", g_stConfInfo.dbName, g_stConfInfo.numOfVgroups); + TAOS_RES* pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + sprintf(sqlStr, "use %s", g_stConfInfo.dbName); + pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + // create row value + g_pRowValue = (char*)taosMemoryCalloc(1, g_stConfInfo.numOfColumn * 16 + 128); + if (NULL == g_pRowValue) { + return -1; + } + + int32_t dataLen = 0; + int32_t sqlLen = 0; + sqlLen += sprintf(sqlStr + sqlLen, "create stable if not exists %s (ts timestamp, ", g_stConfInfo.stbName); + for (int32_t i = 0; i < g_stConfInfo.numOfColumn; i++) { + if (i == g_stConfInfo.numOfColumn - 1) { + sqlLen += sprintf(sqlStr + sqlLen, "c%d int) ", i); + memcpy(g_pRowValue + dataLen, "66778899", strlen("66778899")); + dataLen += strlen("66778899"); + } else { + sqlLen += sprintf(sqlStr + sqlLen, "c%d int, ", i); + memcpy(g_pRowValue + dataLen, "66778899, ", strlen("66778899, ")); + dataLen += strlen("66778899, "); + } + } + sqlLen += sprintf(sqlStr + sqlLen, "tags (t0 int)"); + + pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("failed to create super table %s, reason:%s\n", g_stConfInfo.stbName, taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + for (int32_t i = 0; i < g_stConfInfo.numOfTables; i++) { + sprintf(sqlStr, "create table if not exists %s%d using %s tags(1)", g_stConfInfo.stbName, i, g_stConfInfo.stbName); + pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("failed to create child table %s%d, reason:%s\n", g_stConfInfo.stbName, i, taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + } + + // const char* sql = "select * from tu1"; + sprintf(sqlStr, "create topic test_stb_topic_1 as select ts,c0 from %s", g_stConfInfo.stbName); + /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sqlStr, strlen(sqlStr));*/ + pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + taos_close(pConn); + return 0; +} + +tmq_t* build_consumer() { +#if 0 + char sqlStr[1024] = {0}; + + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + sprintf(sqlStr, "use %s", g_stConfInfo.dbName); + TAOS_RES* pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); +#endif + + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + assert(tmq); + tmq_conf_destroy(conf); + return tmq; +} + +tmq_list_t* build_topic_list() { + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "test_stb_topic_1"); + return topic_list; +} + +void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { + static const int MIN_COMMIT_COUNT = 1000; + + int msg_count = 0; + int32_t err; + + if ((err = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); + return; + } + + while (running) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1); + if (tmqmessage) { + /*msg_process(tmqmessage);*/ + taos_free_result(tmqmessage); + + if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit_sync(tmq, NULL); + } + } + + err = tmq_consumer_close(tmq); + if (err) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLogSize) { + int32_t err; + + if ((err = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); + printf("subscribe err\n"); + return; + } + /*taosSsleep(3);*/ + int32_t batchCnt = 0; + int64_t startTime = taosGetTimestampUs(); + while (running) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 3000); + if (tmqmessage) { + batchCnt++; + if (0 != g_stConfInfo.showMsgFlag) { + /*msg_process(tmqmessage);*/ + } + taos_free_result(tmqmessage); + } else { + break; + } + } + int64_t endTime = taosGetTimestampUs(); + double consumeTime = (double)(endTime - startTime) / 1000000; + + if (batchCnt != totalMsgs) { + printf("%s inserted msgs: %d and consume msgs: %d mismatch %s", GREEN, totalMsgs, batchCnt, NC); + /*exit(-1);*/ + } + + if (0 == g_stConfInfo.simCase) { + printf("consume result: msgs: %d, time used:%.3f second\n", batchCnt, consumeTime); + } else { + printf("{consume success: %d}", totalMsgs); + } + taosFprintfFile(g_fp, "|%10d | %10.3f | %8.2f | %10.2f| %10.2f |\n", batchCnt, consumeTime, + (double)batchCnt / consumeTime, (double)walLogSize / (1024 * 1024.0) / consumeTime, + (double)walLogSize / 1024.0 / batchCnt); + + err = tmq_consumer_close(tmq); + if (err) { + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); + exit(-1); + } +} + +// sync insertion +int32_t syncWriteData() { + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + char sqlStr[1024] = {0}; + sprintf(sqlStr, "use %s", g_stConfInfo.dbName); + TAOS_RES* pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + char* buffer = NULL; + buffer = (char*)taosMemoryMalloc(MAX_SQL_STR_LEN); + if (NULL == buffer) { + return -1; + } + + int32_t totalMsgs = 0; + + int64_t time_counter = g_stConfInfo.startTimestamp; + for (int i = 0; i < g_stConfInfo.totalRowsOfPerTbl;) { + for (int tID = 0; tID <= g_stConfInfo.numOfTables - 1; tID++) { + int inserted = i; + int64_t tmp_time = time_counter; + + int32_t data_len = 0; + data_len += sprintf(buffer + data_len, "insert into %s%d values", g_stConfInfo.stbName, tID); + int k; + for (k = 0; k < g_stConfInfo.batchNumOfRow;) { + data_len += sprintf(buffer + data_len, "(%" PRId64 ", %s) ", tmp_time++, g_pRowValue); + inserted++; + k++; + + if (inserted >= g_stConfInfo.totalRowsOfPerTbl) { + break; + } + + if (data_len > MAX_SQL_STR_LEN - MAX_ROW_STR_LEN) { + break; + } + } + + int code = queryDB(pConn, buffer); + if (0 != code) { + fprintf(stderr, "insert data error!\n"); + taosMemoryFreeClear(buffer); + return -1; + } + + totalMsgs++; + + if (tID == g_stConfInfo.numOfTables - 1) { + i = inserted; + time_counter = tmp_time; + } + } + } + taosMemoryFreeClear(buffer); + return totalMsgs; +} + +// sync insertion +int32_t syncWriteDataByRatio() { + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + char sqlStr[1024] = {0}; + sprintf(sqlStr, "use %s", g_stConfInfo.dbName); + TAOS_RES* pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + char* buffer = NULL; + buffer = (char*)taosMemoryMalloc(MAX_SQL_STR_LEN); + if (NULL == buffer) { + return -1; + } + + int32_t totalMsgs = 0; + + int32_t insertedOfT1 = 0; + int32_t insertedOfT2 = 0; + + int64_t tsOfT1 = g_stConfInfo.startTimestamp; + int64_t tsOfT2 = g_stConfInfo.startTimestamp; + int64_t tmp_time; + + for (;;) { + if ((insertedOfT1 >= g_stConfInfo.totalRowsOfPerTbl) && (insertedOfT2 >= g_stConfInfo.totalRowsOfT2)) { + break; + } + + for (int tID = 0; tID <= g_stConfInfo.numOfTables - 1; tID++) { + if (0 == tID) { + tmp_time = tsOfT1; + if (insertedOfT1 >= g_stConfInfo.totalRowsOfPerTbl) { + continue; + } + } else if (1 == tID) { + tmp_time = tsOfT2; + if (insertedOfT2 >= g_stConfInfo.totalRowsOfT2) { + continue; + } + } + + int32_t data_len = 0; + data_len += sprintf(buffer + data_len, "insert into %s%d values", g_stConfInfo.stbName, tID); + int k; + for (k = 0; k < g_stConfInfo.batchNumOfRow;) { + data_len += sprintf(buffer + data_len, "(%" PRId64 ", %s) ", tmp_time++, g_pRowValue); + k++; + if (0 == tID) { + insertedOfT1++; + if (insertedOfT1 >= g_stConfInfo.totalRowsOfPerTbl) { + break; + } + } else if (1 == tID) { + insertedOfT2++; + if (insertedOfT2 >= g_stConfInfo.totalRowsOfT2) { + break; + } + } + + if (data_len > MAX_SQL_STR_LEN - MAX_ROW_STR_LEN) { + break; + } + } + + int code = queryDB(pConn, buffer); + if (0 != code) { + fprintf(stderr, "insert data error!\n"); + taosMemoryFreeClear(buffer); + return -1; + } + + if (0 == tID) { + tsOfT1 = tmp_time; + } else if (1 == tID) { + tsOfT2 = tmp_time; + } + + totalMsgs++; + } + } + pPrint("expect insert rows: T1[%d] T2[%d], actual insert rows: T1[%d] T2[%d]\n", g_stConfInfo.totalRowsOfPerTbl, + g_stConfInfo.totalRowsOfT2, insertedOfT1, insertedOfT2); + taosMemoryFreeClear(buffer); + return totalMsgs; +} + +void printParaIntoFile() { + // FILE *fp = fopen(g_stConfInfo.resultFileName, "a"); + TdFilePtr pFile = + taosOpenFile(g_stConfInfo.resultFileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_STREAM); + if (NULL == pFile) { + fprintf(stderr, "Failed to open %s for save result\n", g_stConfInfo.resultFileName); + exit(-1); + } + g_fp = pFile; + + time_t tTime = taosGetTimestampSec(); + struct tm tm; + taosLocalTime(&tTime, &tm); + + taosFprintfFile(pFile, "###################################################################\n"); + taosFprintfFile(pFile, "# configDir: %s\n", configDir); + taosFprintfFile(pFile, "# dbName: %s\n", g_stConfInfo.dbName); + taosFprintfFile(pFile, "# stbName: %s\n", g_stConfInfo.stbName); + taosFprintfFile(pFile, "# vnodeWalPath: %s\n", g_stConfInfo.vnodeWalPath); + taosFprintfFile(pFile, "# numOfTables: %d\n", g_stConfInfo.numOfTables); + taosFprintfFile(pFile, "# numOfThreads: %d\n", g_stConfInfo.numOfThreads); + taosFprintfFile(pFile, "# numOfVgroups: %d\n", g_stConfInfo.numOfVgroups); + taosFprintfFile(pFile, "# runMode: %d\n", g_stConfInfo.runMode); + taosFprintfFile(pFile, "# ratio: %f\n", g_stConfInfo.ratio); + taosFprintfFile(pFile, "# numOfColumn: %d\n", g_stConfInfo.numOfColumn); + taosFprintfFile(pFile, "# batchNumOfRow: %d\n", g_stConfInfo.batchNumOfRow); + taosFprintfFile(pFile, "# totalRowsOfPerTbl: %d\n", g_stConfInfo.totalRowsOfPerTbl); + taosFprintfFile(pFile, "# totalRowsOfT2: %d\n", g_stConfInfo.totalRowsOfT2); + taosFprintfFile(pFile, "# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + taosFprintfFile(pFile, "###################################################################\n"); + taosFprintfFile(pFile, + "|-------------------------------insert " + "info-----------------------------|--------------------------------consume " + "info---------------------------------|\n"); + taosFprintfFile(pFile, + "|batch size| insert msgs | insert time(s) | msgs/s | walLogSize(MB) | consume msgs | consume " + "time(s) | msgs/s | MB/s | avg msg size(KB) |\n"); + taosFprintfFile(g_fp, "|%10d", g_stConfInfo.batchNumOfRow); +} + +int main(int32_t argc, char* argv[]) { + parseArgument(argc, argv); + printParaIntoFile(); + + int64_t walLogSize = 0; + + int code; + code = init_env(); + if (code != 0) { + fprintf(stderr, "%% init_env error!\n"); + return -1; + } + + int32_t totalMsgs = 0; + + if (g_stConfInfo.runMode != TMQ_RUN_ONLY_CONSUME) { + int64_t startTs = taosGetTimestampUs(); + if (1 == g_stConfInfo.ratio) { + totalMsgs = syncWriteData(); + } else { + totalMsgs = syncWriteDataByRatio(); + } + + if (totalMsgs <= 0) { + pError("inset data error!\n"); + return -1; + } + int64_t endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + + int32_t totalRows = 0; + if (1 == g_stConfInfo.ratio) { + totalRows = g_stConfInfo.totalRowsOfPerTbl * g_stConfInfo.numOfTables; + } else { + totalRows = g_stConfInfo.totalRowsOfPerTbl * (1 + g_stConfInfo.ratio); + } + + float seconds = delay / 1000000.0; + float rowsSpeed = totalRows / seconds; + float msgsSpeed = totalMsgs / seconds; + + if ((0 == g_stConfInfo.simCase) && (strlen(g_stConfInfo.vnodeWalPath))) { + walLogSize = getDirectorySize(g_stConfInfo.vnodeWalPath); + if (walLogSize <= 0) { + printf("%s size incorrect!", g_stConfInfo.vnodeWalPath); + exit(-1); + } else { + pPrint(".log file size in vnode2/wal: %.3f MBytes\n", (double)walLogSize / (1024 * 1024.0)); + } + } + + if (0 == g_stConfInfo.simCase) { + pPrint("insert result: %d rows, %d msgs, time:%.3f sec, speed:%.1f rows/second, %.1f msgs/second\n", totalRows, + totalMsgs, seconds, rowsSpeed, msgsSpeed); + } + taosFprintfFile(g_fp, "|%10d | %10.3f | %8.2f | %10.3f ", totalMsgs, seconds, msgsSpeed, + (double)walLogSize / (1024 * 1024.0)); + } + + if (g_stConfInfo.runMode == TMQ_RUN_ONLY_INSERT) { + return 0; + } + + tmq_t* tmq = build_consumer(); + tmq_list_t* topic_list = build_topic_list(); + if ((NULL == tmq) || (NULL == topic_list)) { + return -1; + } + + perf_loop(tmq, topic_list, totalMsgs, walLogSize); + + taosMemoryFreeClear(g_pRowValue); + taosFprintfFile(g_fp, "\n"); + taosCloseFile(&g_fp); + return 0; +} + diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c new file mode 100644 index 0000000000..71b31ba107 --- /dev/null +++ b/utils/test/c/tmqSim.c @@ -0,0 +1,1520 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "taos.h" +#include "taosdef.h" +#include "taoserror.h" +#include "tlog.h" +#include "types.h" + +#define GREEN "\033[1;32m" +#define NC "\033[0m" +#define min(a, b) (((a) < (b)) ? (a) : (b)) + +#define MAX_SQL_STR_LEN (1024 * 1024) +#define MAX_ROW_STR_LEN (16 * 1024) +#define MAX_CONSUMER_THREAD_CNT (16) +#define MAX_VGROUP_CNT (32) +#define SEND_TIME_UNIT 10 // ms +#define MAX_SQL_LEN 1048576 + +typedef enum { + NOTIFY_CMD_START_CONSUM, + NOTIFY_CMD_START_COMMIT, + NOTIFY_CMD_ID_BUTT, +} NOTIFY_CMD_ID; + +typedef enum enumQUERY_TYPE { + NO_INSERT_TYPE, + INSERT_TYPE, + QUERY_TYPE_BUT +} QUERY_TYPE; + +typedef struct { + TdThread thread; + int32_t consumerId; + + int32_t ifManualCommit; + // int32_t autoCommitIntervalMs; // 1000 ms + // char autoCommit[8]; // true, false + // char autoOffsetRest[16]; // none, earliest, latest + + TdFilePtr pConsumeRowsFile; + TdFilePtr pConsumeMetaFile; + int32_t ifCheckData; + int64_t expectMsgCnt; + + int64_t consumeMsgCnt; + int64_t consumeRowCnt; + int64_t consumeLen; + int32_t checkresult; + + char topicString[1024]; + char keyString[1024]; + + int32_t numOfTopic; + char topics[32][64]; + + int32_t numOfKey; + char key[32][64]; + char value[32][64]; + + tmq_t* tmq; + tmq_list_t* topicList; + + int32_t numOfVgroups; + int32_t rowsOfPerVgroups[MAX_VGROUP_CNT][2]; // [i][0]: vgroup id, [i][1]: rows of consume + int64_t ts; + + TAOS* taos; + + // below parameters is used by omb test + int32_t producerRate; // unit: msgs/s + int64_t totalProduceMsgs; + int64_t totalMsgsLen; + +} SThreadInfo; + +typedef struct { + // input from argvs + char cdbName[32]; + char dbName[64]; + int32_t showMsgFlag; + int32_t showRowFlag; + int32_t saveRowFlag; + int32_t consumeDelay; // unit s + int32_t numOfThread; + int32_t useSnapshot; + int64_t nowTime; + SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; + + SThreadInfo stProdThreads[MAX_CONSUMER_THREAD_CNT]; + + // below parameters is used by omb test + char topic[64]; + int32_t producers; + int32_t producerRate; + int32_t runDurationMinutes; + int32_t batchSize; + int32_t payloadLen; +} SConfInfo; + +static SConfInfo g_stConfInfo; +TdFilePtr g_fp = NULL; +static int running = 1; +char* g_payload = NULL; + +// char* g_pRowValue = NULL; +// TdFilePtr g_fp = NULL; + +static void printHelp() { + char indent[10] = " "; + printf("Used to test the tmq feature with sim cases\n"); + + printf("%s%s\n", indent, "-c"); + printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); + printf("%s%s\n", indent, "-d"); + printf("%s%s%s\n", indent, indent, "The name of the database for cosumer, no default "); + printf("%s%s\n", indent, "-g"); + printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%d\n", indent, indent, "showRowFlag, default is ", g_stConfInfo.showRowFlag); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%d\n", indent, indent, "saveRowFlag, default is ", g_stConfInfo.saveRowFlag); + printf("%s%s\n", indent, "-y"); + printf("%s%s%s%ds\n", indent, indent, "consume delay, default is ", g_stConfInfo.consumeDelay); + printf("%s%s\n", indent, "-e"); + printf("%s%s%s%d\n", indent, indent, "snapshot, default is ", g_stConfInfo.useSnapshot); + + printf("%s%s\n", indent, "-t"); + printf("%s%s%s\n", indent, indent, "topic name, default is null"); + + printf("%s%s\n", indent, "-x"); + printf("%s%s%s\n", indent, indent, "consume thread number, default is 1"); + + + printf("%s%s\n", indent, "-l"); + printf("%s%s%s\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes); + printf("%s%s\n", indent, "-p"); + printf("%s%s%s\n", indent, indent, "producer thread number, default is 0"); + printf("%s%s\n", indent, "-b"); + printf("%s%s%s\n", indent, indent, "batch size, default is 1"); + printf("%s%s\n", indent, "-i"); + printf("%s%s%s\n", indent, indent, "produce rate unit is msgs /s, default is 100000"); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s\n", indent, indent, "payload len unit is byte, default is 1000"); + + + exit(EXIT_SUCCESS); +} + +char* getCurrentTimeString(char* timeString) { + time_t tTime = taosGetTimestampSec(); + struct tm tm; + taosLocalTime(&tTime, &tm); + sprintf(timeString, "%d-%02d-%02d %02d:%02d:%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, + tm.tm_min, tm.tm_sec); + + return timeString; +} + +static void tmqStop(int signum, void* info, void* ctx) { + running = 0; + char tmpString[128]; + taosFprintfFile(g_fp, "%s tmqStop() receive stop signal[%d]\n", getCurrentTimeString(tmpString), signum); +} + +static void tmqSetSignalHandle() { taosSetSignal(SIGINT, tmqStop); } + +void initLogFile() { + char filename[256]; + char tmpString[128]; + + pid_t process_id = getpid(); + + if (0 != strlen(g_stConfInfo.topic)) { + sprintf(filename, "/tmp/tmqlog-%d-%s.txt", process_id, getCurrentTimeString(tmpString)); + } else { + sprintf(filename, "%s/../log/tmqlog-%d-%s.txt", configDir, process_id, getCurrentTimeString(tmpString)); + } +#ifdef WINDOWS + for (int i = 2; i < sizeof(filename); i++) { + if (filename[i] == ':') filename[i] = '-'; + if (filename[i] == '\0') break; + } +#endif + TdFilePtr pFile = taosOpenFile(filename, TD_FILE_TEXT | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + if (NULL == pFile) { + fprintf(stderr, "Failed to open %s for save result\n", filename); + exit(-1); + } + g_fp = pFile; +} + +void saveConfigToLogFile() { + taosFprintfFile(g_fp, "###################################################################\n"); + taosFprintfFile(g_fp, "# configDir: %s\n", configDir); + taosFprintfFile(g_fp, "# dbName: %s\n", g_stConfInfo.dbName); + taosFprintfFile(g_fp, "# cdbName: %s\n", g_stConfInfo.cdbName); + taosFprintfFile(g_fp, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); + taosFprintfFile(g_fp, "# showRowFlag: %d\n", g_stConfInfo.showRowFlag); + taosFprintfFile(g_fp, "# saveRowFlag: %d\n", g_stConfInfo.saveRowFlag); + taosFprintfFile(g_fp, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); + taosFprintfFile(g_fp, "# numOfThread: %d\n", g_stConfInfo.numOfThread); + + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); + // taosFprintfFile(g_fp, " auto commit: %s\n", g_stConfInfo.stThreads[i].autoCommit); + // taosFprintfFile(g_fp, " auto commit interval ms: %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs); + // taosFprintfFile(g_fp, " auto offset rest: %s\n", g_stConfInfo.stThreads[i].autoOffsetRest); + taosFprintfFile(g_fp, " Topics: "); + for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) { + taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]); + } + taosFprintfFile(g_fp, "\n"); + taosFprintfFile(g_fp, " Key: "); + for (int k = 0; k < g_stConfInfo.stThreads[i].numOfKey; k++) { + taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]); + } + taosFprintfFile(g_fp, "\n"); + taosFprintfFile(g_fp, " expect rows: %d\n", g_stConfInfo.stThreads[i].expectMsgCnt); + } + + char tmpString[128]; + taosFprintfFile(g_fp, "# Test time: %s\n", getCurrentTimeString(tmpString)); + taosFprintfFile(g_fp, "###################################################################\n"); +} + +void parseArgument(int32_t argc, char* argv[]) { + memset(&g_stConfInfo, 0, sizeof(SConfInfo)); + g_stConfInfo.showMsgFlag = 0; + g_stConfInfo.showRowFlag = 0; + g_stConfInfo.saveRowFlag = 0; + g_stConfInfo.consumeDelay = 5; + g_stConfInfo.numOfThread = 1; + g_stConfInfo.batchSize = 1; + g_stConfInfo.producers = 0; + + g_stConfInfo.nowTime = taosGetTimestampMs(); + + for (int32_t i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-d") == 0) { + strcpy(g_stConfInfo.dbName, argv[++i]); + } else if (strcmp(argv[i], "-w") == 0) { + strcpy(g_stConfInfo.cdbName, argv[++i]); + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-g") == 0) { + g_stConfInfo.showMsgFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + g_stConfInfo.showRowFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + g_stConfInfo.saveRowFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-y") == 0) { + g_stConfInfo.consumeDelay = atol(argv[++i]); + } else if (strcmp(argv[i], "-e") == 0) { + g_stConfInfo.useSnapshot = atol(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + char tmpBuf[56]; + strcpy(tmpBuf, argv[++i]); + sprintf(g_stConfInfo.topic, "`%s`", tmpBuf); + } else if (strcmp(argv[i], "-x") == 0) { + g_stConfInfo.numOfThread = atol(argv[++i]); + } else if (strcmp(argv[i], "-l") == 0) { + g_stConfInfo.runDurationMinutes = atol(argv[++i]); + } else if (strcmp(argv[i], "-p") == 0) { + g_stConfInfo.producers = atol(argv[++i]); + } else if (strcmp(argv[i], "-b") == 0) { + g_stConfInfo.batchSize = atol(argv[++i]); + } else if (strcmp(argv[i], "-i") == 0) { + g_stConfInfo.producerRate = atol(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + g_stConfInfo.payloadLen = atol(argv[++i]); + } else { + pError("%s unknow para: %s %s", GREEN, argv[++i], NC); + exit(-1); + } + } + + g_payload = taosMemoryCalloc(g_stConfInfo.payloadLen + 1, 1); + if (NULL == g_payload) { + pPrint("%s failed to malloc for payload %s", GREEN, NC); + exit(-1); + } + + for (int32_t i = 0; i < g_stConfInfo.payloadLen; i++) { + strcpy(&g_payload[i], "a"); + } + + initLogFile(); + + taosFprintfFile(g_fp, "====parseArgument() success\n"); + +#if 1 + pPrint("%s configDir:%s %s", GREEN, configDir, NC); + pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC); + pPrint("%s cdbName:%s %s", GREEN, g_stConfInfo.cdbName, NC); + pPrint("%s consumeDelay:%d %s", GREEN, g_stConfInfo.consumeDelay, NC); + pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); + pPrint("%s showRowFlag:%d %s", GREEN, g_stConfInfo.showRowFlag, NC); + pPrint("%s saveRowFlag:%d %s", GREEN, g_stConfInfo.saveRowFlag, NC); + + pPrint("%s snapshot:%d %s", GREEN, g_stConfInfo.useSnapshot, NC); + + pPrint("%s omb topic:%s %s", GREEN, g_stConfInfo.topic, NC); + pPrint("%s numOfThread:%d %s", GREEN, g_stConfInfo.numOfThread, NC); +#endif +} + +void splitStr(char** arr, char* str, const char* del) { + char* s = strtok(str, del); + while (s != NULL) { + *arr++ = s; + s = strtok(NULL, del); + } +} + +void ltrim(char* str) { + if (str == NULL || *str == '\0') { + return; + } + int len = 0; + char* p = str; + while (*p != '\0' && isspace(*p)) { + ++p; + ++len; + } + memmove(str, p, strlen(str) - len + 1); + // return str; +} + +int queryDB(TAOS* taos, char* command) { + int retryCnt = 10; + int code; + TAOS_RES* pRes; + + while (retryCnt--) { + pRes = taos_query(taos, command); + code = taos_errno(pRes); + if (code != 0) { + taosSsleep(1); + taos_free_result(pRes); + continue; + } + taos_free_result(pRes); + return 0; + } + + pError("failed to reason:%s, sql: %s", tstrerror(code), command); + taos_free_result(pRes); + return -1; +} + + +void addRowsToVgroupId(SThreadInfo* pInfo, int32_t vgroupId, int32_t rows) { + int32_t i; + for (i = 0; i < pInfo->numOfVgroups; i++) { + if (vgroupId == pInfo->rowsOfPerVgroups[i][0]) { + pInfo->rowsOfPerVgroups[i][1] += rows; + return; + } + } + + pInfo->rowsOfPerVgroups[pInfo->numOfVgroups][0] = vgroupId; + pInfo->rowsOfPerVgroups[pInfo->numOfVgroups][1] += rows; + pInfo->numOfVgroups++; + + taosFprintfFile(g_fp, "consume id %d, add one new vogroup id: %d\n", pInfo->consumerId, vgroupId); + if (pInfo->numOfVgroups > MAX_VGROUP_CNT) { + taosFprintfFile(g_fp, "====consume id %d, vgroup num %d over than 32. new vgroupId: %d\n", pInfo->consumerId, + pInfo->numOfVgroups, vgroupId); + taosCloseFile(&g_fp); + exit(-1); + } +} + +TAOS* createNewTaosConnect() { + TAOS* taos = NULL; + int32_t retryCnt = 10; + + while (retryCnt--) { + TAOS* taos = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (NULL != taos) { + return taos; + } + taosSsleep(1); + } + + taosFprintfFile(g_fp, "taos_connect() fail\n"); + return NULL; +} + + +int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) { + char sqlStr[1100] = {0}; + + if (strlen(buf) > 1024) { + taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", strlen(buf)); + taosCloseFile(&g_fp); + return -1; + } + + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (pConn == NULL) { + taosFprintfFile(g_fp, "taos_connect() fail, can not save consume result to main script\n"); + return -1; + } + + sprintf(sqlStr, "insert into %s.content_%d values (%" PRId64 ", \'%s\')", g_stConfInfo.cdbName, pInfo->consumerId, + pInfo->ts++, buf); + int retCode = queryDB(pConn, sqlStr); + if (retCode != 0) { + taosFprintfFile(g_fp, "error in save consume content\n"); + taosCloseFile(&g_fp); + taos_close(pConn); + exit(-1); + } + + taos_close(pConn); + + return 0; +} + +static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) { + // if (shell.args.is_raw_time) { + // sprintf(buf, "%" PRId64, val); + // return buf; + // } + + time_t tt; + int32_t ms = 0; + if (precision == TSDB_TIME_PRECISION_NANO) { + tt = (time_t)(val / 1000000000); + ms = val % 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + tt = (time_t)(val / 1000000); + ms = val % 1000000; + } else { + tt = (time_t)(val / 1000); + ms = val % 1000; + } + + if (tt <= 0 && ms < 0) { + tt--; + if (precision == TSDB_TIME_PRECISION_NANO) { + ms += 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + ms += 1000000; + } else { + ms += 1000; + } + } + + struct tm ptm; + taosLocalTime(&tt, &ptm); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); + + if (precision == TSDB_TIME_PRECISION_NANO) { + sprintf(buf + pos, ".%09d", ms); + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + sprintf(buf + pos, ".%06d", ms); + } else { + sprintf(buf + pos, ".%03d", ms); + } + + return buf; +} + +static void shellDumpFieldToFile(TdFilePtr pFile, const char* val, TAOS_FIELD* field, int32_t length, + int32_t precision) { + if (val == NULL) { + return; + } + + int n; + char buf[TSDB_MAX_BYTES_PER_ROW]; + switch (field->type) { + case TSDB_DATA_TYPE_BOOL: + taosFprintfFile(pFile, "%d", ((((int32_t)(*((char*)val))) == 1) ? 1 : 0)); + break; + case TSDB_DATA_TYPE_TINYINT: + taosFprintfFile(pFile, "%d", *((int8_t*)val)); + break; + case TSDB_DATA_TYPE_UTINYINT: + taosFprintfFile(pFile, "%u", *((uint8_t*)val)); + break; + case TSDB_DATA_TYPE_SMALLINT: + taosFprintfFile(pFile, "%d", *((int16_t*)val)); + break; + case TSDB_DATA_TYPE_USMALLINT: + taosFprintfFile(pFile, "%u", *((uint16_t*)val)); + break; + case TSDB_DATA_TYPE_INT: + taosFprintfFile(pFile, "%d", *((int32_t*)val)); + break; + case TSDB_DATA_TYPE_UINT: + taosFprintfFile(pFile, "%u", *((uint32_t*)val)); + break; + case TSDB_DATA_TYPE_BIGINT: + taosFprintfFile(pFile, "%" PRId64, *((int64_t*)val)); + break; + case TSDB_DATA_TYPE_UBIGINT: + taosFprintfFile(pFile, "%" PRIu64, *((uint64_t*)val)); + break; + case TSDB_DATA_TYPE_FLOAT: + taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val)); + break; + case TSDB_DATA_TYPE_DOUBLE: + n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val)); + if (n > TMAX(25, length)) { + taosFprintfFile(pFile, "%*.15e", length, GET_DOUBLE_VAL(val)); + } else { + taosFprintfFile(pFile, "%s", buf); + } + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_JSON: + { + char quotationStr[2]; + int32_t bufIndex = 0; + quotationStr[0] = 0; + quotationStr[1] = 0; + for (int32_t i = 0; i < length; i++) { + buf[bufIndex] = val[i]; + bufIndex++; + if (val[i] == '\"') { + buf[bufIndex] = val[i]; + bufIndex++; + quotationStr[0] = '\"'; + } + if (val[i] == ',') { + quotationStr[0] = '\"'; + } + } + buf[bufIndex] = 0; + if (length == 0) { + quotationStr[0] = '\"'; + } + + taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr); + } + break; + case TSDB_DATA_TYPE_TIMESTAMP: + shellFormatTimestamp(buf, *(int64_t*)val, precision); + taosFprintfFile(pFile, "%s", buf); + break; + default: + break; + } +} + +static void dumpToFileForCheck(TdFilePtr pFile, TAOS_ROW row, TAOS_FIELD* fields, int32_t* length, int32_t num_fields, + int32_t precision) { + for (int32_t i = 0; i < num_fields; i++) { + if (i > 0) { + taosFprintfFile(pFile, ","); + } + shellDumpFieldToFile(pFile, (const char*)row[i], fields + i, length[i], precision); + } + taosFprintfFile(pFile, "\n"); +} + +static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) { + char buf[1024]; + int32_t totalRows = 0; + + // printf("topic: %s\n", tmq_get_topic_name(msg)); + int32_t vgroupId = tmq_get_vgroup_id(msg); + const char* dbName = tmq_get_db_name(msg); + + taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", + tmq_get_topic_name(msg), vgroupId); + + while (1) { + TAOS_ROW row = taos_fetch_row(msg); + + if (row == NULL) break; + + TAOS_FIELD* fields = taos_fetch_fields(msg); + int32_t numOfFields = taos_field_count(msg); + int32_t* length = taos_fetch_lengths(msg); + int32_t precision = taos_result_precision(msg); + const char* tbName = tmq_get_table_name(msg); + +#if 0 + // get schema + //============================== stub =================================================// + for (int32_t i = 0; i < numOfFields; i++) { + taosFprintfFile(g_fp, "%02d: name: %s, type: %d, len: %d\n", i, fields[i].name, fields[i].type, fields[i].bytes); + } + //============================== stub =================================================// +#endif + + dumpToFileForCheck(pInfo->pConsumeRowsFile, row, fields, length, numOfFields, precision); + + taos_print_row(buf, row, fields, numOfFields); + + if (0 != g_stConfInfo.showRowFlag) { + taosFprintfFile(g_fp, "tbname:%s, rows[%d]: %s\n", (tbName != NULL ? tbName : "null table"), totalRows, buf); + // if (0 != g_stConfInfo.saveRowFlag) { + // saveConsumeContentToTbl(pInfo, buf); + // } + } + + totalRows++; + } + + addRowsToVgroupId(pInfo, vgroupId, totalRows); + + return totalRows; +} + + +static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) { + char buf[1024]; + int32_t totalRows = 0; + + // printf("topic: %s\n", tmq_get_topic_name(msg)); + int32_t vgroupId = tmq_get_vgroup_id(msg); + const char* dbName = tmq_get_db_name(msg); + + taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", + tmq_get_topic_name(msg), vgroupId); + + { + tmq_raw_data raw = {0}; + int32_t code = tmq_get_raw(msg, &raw); + + if(code == TSDB_CODE_SUCCESS){ + int retCode = queryDB(pInfo->taos, "use metadb"); + if (retCode != 0) { + taosFprintfFile(g_fp, "error when use metadb\n"); + taosCloseFile(&g_fp); + exit(-1); + } + taosFprintfFile(g_fp, "raw:%p\n", &raw); + + tmq_write_raw(pInfo->taos, raw); + } + + char* result = tmq_get_json_meta(msg); + if(result){ + //printf("meta result: %s\n", result); + taosFprintfFile(pInfo->pConsumeMetaFile, "%s\n", result); + taosMemoryFree(result); + } + } + + totalRows++; + + return totalRows; +} + +static void appNothing(void* param, TAOS_RES* res, int32_t numOfRows) {} + +int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) { + char sqlStr[1024] = {0}; + + // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int + sprintf(sqlStr, "insert into %s.notifyinfo values (%" PRId64 ", %d, %d)", g_stConfInfo.cdbName, atomic_fetch_add_64(&g_stConfInfo.nowTime, 1), cmdId, + pInfo->consumerId); + + taos_query_a(pInfo->taos, sqlStr, appNothing, NULL); + + taosFprintfFile(g_fp, "notifyMainScript success, sql: %s\n", sqlStr); + + return 0; +} + +static int32_t g_once_commit_flag = 0; +static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + taosFprintfFile(g_fp, "tmq_commit_cb_print() commit %d\n", code); + + if (0 == g_once_commit_flag) { + g_once_commit_flag = 1; + notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT); + } + + char tmpString[128]; + taosFprintfFile(g_fp, "%s tmq_commit_cb_print() be called\n", getCurrentTimeString(tmpString)); +} + +void build_consumer(SThreadInfo* pInfo) { + tmq_conf_t* conf = tmq_conf_new(); + + // tmq_conf_set(conf, "td.connect.ip", "localhost"); + // tmq_conf_set(conf, "td.connect.port", "6030"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + + // tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, pInfo); + + // tmq_conf_set(conf, "group.id", "cgrp1"); + for (int32_t i = 0; i < pInfo->numOfKey; i++) { + tmq_conf_set(conf, pInfo->key[i], pInfo->value[i]); + } + + tmq_conf_set(conf, "msg.with.table.name", "true"); + + // tmq_conf_set(conf, "client.id", "c-001"); + + // tmq_conf_set(conf, "enable.auto.commit", "true"); + // tmq_conf_set(conf, "enable.auto.commit", "false"); + + // tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); + + // tmq_conf_set(conf, "auto.offset.reset", "none"); + // tmq_conf_set(conf, "auto.offset.reset", "earliest"); + // tmq_conf_set(conf, "auto.offset.reset", "latest"); + // + if (g_stConfInfo.useSnapshot) { + tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + } + + pInfo->tmq = tmq_consumer_new(conf, NULL, 0); + + tmq_conf_destroy(conf); + + return; +} + +void build_topic_list(SThreadInfo* pInfo) { + pInfo->topicList = tmq_list_new(); + // tmq_list_append(topic_list, "test_stb_topic_1"); + for (int32_t i = 0; i < pInfo->numOfTopic; i++) { + tmq_list_append(pInfo->topicList, pInfo->topics[i]); + } + return; +} + +int32_t saveConsumeResult(SThreadInfo* pInfo) { + char sqlStr[1024] = {0}; + // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int + sprintf(sqlStr, "insert into %s.consumeresult values (%" PRId64 ", %d, %" PRId64 ", %" PRId64 ", %d)", + g_stConfInfo.cdbName, atomic_fetch_add_64(&g_stConfInfo.nowTime, 1), pInfo->consumerId, pInfo->consumeMsgCnt, + pInfo->consumeRowCnt, pInfo->checkresult); + + char tmpString[128]; + taosFprintfFile(g_fp, "%s, consume id %d result: %s\n", getCurrentTimeString(tmpString), pInfo->consumerId, sqlStr); + + int retCode = queryDB(pInfo->taos, sqlStr); + if (retCode != 0) { + taosFprintfFile(g_fp, "consume id %d error in save consume result\n", pInfo->consumerId); + return -1; + } + + return 0; +} + +void loop_consume(SThreadInfo* pInfo) { + int32_t code; + + int32_t once_flag = 0; + + int64_t totalMsgs = 0; + int64_t totalRows = 0; + + char tmpString[128]; + taosFprintfFile(g_fp, "%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), + pInfo->consumerId); + + pInfo->ts = taosGetTimestampMs(); + + if (pInfo->ifCheckData) { + char filename[256] = {0}; + char tmpString[128]; + // sprintf(filename, "%s/../log/consumerid_%d_%s.txt", configDir, pInfo->consumerId, + // getCurrentTimeString(tmpString)); + sprintf(filename, "%s/../log/consumerid_%d.txt", configDir, pInfo->consumerId); + pInfo->pConsumeRowsFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + + sprintf(filename, "%s/../log/meta_consumerid_%d.txt", configDir, pInfo->consumerId); + pInfo->pConsumeMetaFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + + if (pInfo->pConsumeRowsFile == NULL || pInfo->pConsumeMetaFile == NULL) { + taosFprintfFile(g_fp, "%s create file fail for save rows or save meta\n", getCurrentTimeString(tmpString)); + return; + } + } + + int64_t lastTotalMsgs = 0; + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); + + int32_t consumeDelay = g_stConfInfo.consumeDelay == -1 ? -1 : (g_stConfInfo.consumeDelay * 1000); + while (running) { + TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, consumeDelay); + if (tmqMsg) { + if (0 != g_stConfInfo.showMsgFlag) { + tmq_res_t msgType = tmq_get_res_type(tmqMsg); + if (msgType == TMQ_RES_TABLE_META) { + totalRows += meta_msg_process(tmqMsg, pInfo, totalMsgs); + } else if (msgType == TMQ_RES_DATA) + totalRows += data_msg_process(tmqMsg, pInfo, totalMsgs); + } + + taos_free_result(tmqMsg); + + totalMsgs++; + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 10 * 1000) { + taosFprintfFile( + g_fp, "consumer id %d has currently poll total msgs: %" PRId64 ", period rate: %.3f msgs/second\n", + pInfo->consumerId, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0 / (currentPrintTime - lastPrintTime)); + lastPrintTime = currentPrintTime; + lastTotalMsgs = totalMsgs; + } + + if (0 == once_flag) { + once_flag = 1; + notifyMainScript(pInfo, NOTIFY_CMD_START_CONSUM); + } + + if ((totalRows >= pInfo->expectMsgCnt) || (totalMsgs >= pInfo->expectMsgCnt)) { + char tmpString[128]; + taosFprintfFile(g_fp, "%s over than expect rows, so break consume\n", getCurrentTimeString(tmpString)); + break; + } + } else { + char tmpString[128]; + taosFprintfFile(g_fp, "%s no poll more msg when time over, break consume\n", getCurrentTimeString(tmpString)); + break; + } + } + + if (0 == running) { + taosFprintfFile(g_fp, "receive stop signal and not continue consume\n"); + } + + pInfo->consumeMsgCnt = totalMsgs; + pInfo->consumeRowCnt = totalRows; + + taosFprintfFile(g_fp, "==== consumerId: %d, consumeMsgCnt: %" PRId64 ", consumeRowCnt: %" PRId64 "\n", + pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt); +} + +void* consumeThreadFunc(void* param) { + SThreadInfo* pInfo = (SThreadInfo*)param; + + pInfo->taos = createNewTaosConnect(); + if (pInfo->taos == NULL) { + taosFprintfFile(g_fp, "taos_connect() fail, can not notify and save consume result to main scripte\n"); + return NULL; + } + + build_consumer(pInfo); + build_topic_list(pInfo); + if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { + taosFprintfFile(g_fp, "create consumer fail! tmq is null or topicList is null\n"); + taos_close(pInfo->taos); + pInfo->taos = NULL; + return NULL; + } + + int32_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); + if (err != 0) { + pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_subscribe() fail! reason: %s\n", tmq_err2str(err)); + taos_close(pInfo->taos); + pInfo->taos = NULL; + return NULL; + } + + tmq_list_destroy(pInfo->topicList); + pInfo->topicList = NULL; + + loop_consume(pInfo); + + if (pInfo->ifManualCommit) { + pPrint("tmq_commit() manual commit when consume end.\n"); + /*tmq_commit(pInfo->tmq, NULL, 0);*/ + tmq_commit_sync(pInfo->tmq, NULL); + taosFprintfFile(g_fp, "tmq_commit() manual commit over.\n"); + pPrint("tmq_commit() manual commit over.\n"); + } + + err = tmq_unsubscribe(pInfo->tmq); + if (err != 0) { + pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_unsubscribe()! reason: %s\n", tmq_err2str(err)); + } + + err = tmq_consumer_close(pInfo->tmq); + if (err != 0) { + pError("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_consumer_close()! reason: %s\n", tmq_err2str(err)); + } + pInfo->tmq = NULL; + + // save consume result into consumeresult table + saveConsumeResult(pInfo); + + // save rows from per vgroup + taosFprintfFile(g_fp, "======== consumerId: %d, consume rows from per vgroups ========\n", pInfo->consumerId); + for (int32_t i = 0; i < pInfo->numOfVgroups; i++) { + taosFprintfFile(g_fp, "vgroups: %04d, rows: %d\n", pInfo->rowsOfPerVgroups[i][0], pInfo->rowsOfPerVgroups[i][1]); + } + + taos_close(pInfo->taos); + pInfo->taos = NULL; + + return NULL; +} + +void parseConsumeInfo() { + char* token; + const char delim[2] = ","; + const char ch = ':'; + + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + token = strtok(g_stConfInfo.stThreads[i].topicString, delim); + while (token != NULL) { + // printf("%s\n", token ); + strcpy(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic], token); + ltrim(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic]); + // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); + g_stConfInfo.stThreads[i].numOfTopic++; + + token = strtok(NULL, delim); + } + + token = strtok(g_stConfInfo.stThreads[i].keyString, delim); + while (token != NULL) { + // printf("%s\n", token ); + { + char* pstr = token; + ltrim(pstr); + char* ret = strchr(pstr, ch); + memcpy(g_stConfInfo.stThreads[i].key[g_stConfInfo.stThreads[i].numOfKey], pstr, ret - pstr); + strcpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1); + // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], + // g_stConfInfo.value[g_stConfInfo.numOfKey]); + g_stConfInfo.stThreads[i].numOfKey++; + } + + token = strtok(NULL, delim); + } + } +} + +int32_t getConsumeInfo() { + char sqlStr[1024] = {0}; + + TAOS* pConn = createNewTaosConnect(); + if (pConn == NULL) { + taosFprintfFile(g_fp, "taos_connect() fail, can not get consume info for start consumer\n"); + return -1; + } + + sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.cdbName); + TAOS_RES *pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + taosFprintfFile(g_fp, "error in get consumeinfo for %s\n", taos_errstr(pRes)); + taosCloseFile(&g_fp); + taos_free_result(pRes); + taos_close(pConn); + return -1; + } + + TAOS_ROW row = NULL; + int num_fields = taos_num_fields(pRes); + TAOS_FIELD* fields = taos_fetch_fields(pRes); + + // schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, + // ifcheckdata int + + int32_t numOfThread = 0; + while ((row = taos_fetch_row(pRes))) { + int32_t* lengths = taos_fetch_lengths(pRes); + + // set default value + // g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000; + // memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true")); + // memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast")); + + for (int i = 0; i < num_fields; ++i) { + if (row[i] == NULL || 0 == i) { + continue; + } + + if ((1 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { + g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t*)row[i]); + } else if ((2 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { + memcpy(g_stConfInfo.stThreads[numOfThread].topicString, row[i], lengths[i]); + } else if ((3 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { + memcpy(g_stConfInfo.stThreads[numOfThread].keyString, row[i], lengths[i]); + } else if ((4 == i) && (fields[i].type == TSDB_DATA_TYPE_BIGINT)) { + g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]); + } else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { + g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]); + } else if ((6 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { + g_stConfInfo.stThreads[numOfThread].ifManualCommit = *((int32_t*)row[i]); + } + } + numOfThread++; + } + g_stConfInfo.numOfThread = numOfThread; + + taos_free_result(pRes); + + parseConsumeInfo(); + taos_close(pConn); + + return 0; +} + + +static int32_t omb_data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex, int64_t* lenOfRows) { + char buf[16*1024]; + int32_t totalRows = 0; + int32_t totalLen = 0; + + // printf("topic: %s\n", tmq_get_topic_name(msg)); + //int32_t vgroupId = tmq_get_vgroup_id(msg); + //const char* dbName = tmq_get_db_name(msg); + + //taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + //taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", + // tmq_get_topic_name(msg), vgroupId); + + while (1) { + TAOS_ROW row = taos_fetch_row(msg); + + if (row == NULL) break; + + TAOS_FIELD* fields = taos_fetch_fields(msg); + int32_t numOfFields = taos_field_count(msg); + //int32_t* length = taos_fetch_lengths(msg); + //int32_t precision = taos_result_precision(msg); + //const char* tbName = tmq_get_table_name(msg); + + taos_print_row(buf, row, fields, numOfFields); + totalLen += strlen(buf); + totalRows++; + } + + *lenOfRows = totalLen; + return totalRows; +} + +void omb_loop_consume(SThreadInfo* pInfo) { + int32_t code; + + int32_t once_flag = 0; + + int64_t totalMsgs = 0; + int64_t totalRows = 0; + + char tmpString[128]; + taosFprintfFile(g_fp, "%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), + pInfo->consumerId); + printf("%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), + pInfo->consumerId); + + pInfo->ts = taosGetTimestampMs(); + + int64_t lastTotalMsgs = 0; + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); + + int64_t totalLenOfMsg = 0; + int64_t lastTotalLenOfMsg = 0; + int32_t consumeDelay = g_stConfInfo.consumeDelay == -1 ? -1 : (g_stConfInfo.consumeDelay * 1000); + while (running) { + TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, consumeDelay); + if (tmqMsg) { + int64_t lenOfMsg = 0; + totalRows += omb_data_msg_process(tmqMsg, pInfo, totalMsgs, &lenOfMsg); + totalLenOfMsg += lenOfMsg; + taos_free_result(tmqMsg); + totalMsgs++; + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 10 * 1000) { + int64_t currentLenOfMsg = totalLenOfMsg - lastTotalLenOfMsg; + int64_t deltaTime = currentPrintTime - lastPrintTime; + printf("consumer id %d has currently cons total rows: %" PRId64 ", msgs: %" PRId64 ", rate: %.3f msgs/s, %.1f MB/s\n", + pInfo->consumerId, totalRows, totalMsgs, + (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime, + currentLenOfMsg*1000.0/(1024*1024)/deltaTime); + + taosFprintfFile( + g_fp, "consumer id %d has currently poll total msgs: %" PRId64 ", period cons rate: %.3f msgs/s, %.1f MB/s\n", + pInfo->consumerId, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime, currentLenOfMsg*1000.0/deltaTime); + lastPrintTime = currentPrintTime; + lastTotalMsgs = totalMsgs; + lastTotalLenOfMsg = totalLenOfMsg; + } + } else { + char tmpString[128]; + taosFprintfFile(g_fp, "%s no poll more msg when time over, break consume\n", getCurrentTimeString(tmpString)); + printf("%s no poll more msg when time over, break consume\n", getCurrentTimeString(tmpString)); + int64_t currentPrintTime = taosGetTimestampMs(); + int64_t currentLenOfMsg = totalLenOfMsg - lastTotalLenOfMsg; + int64_t deltaTime = currentPrintTime - lastPrintTime; + printf("consumer id %d has currently cons total rows: %" PRId64 ", msgs: %" PRId64 ", rate: %.3f msgs/s, %.1f MB/s\n", + pInfo->consumerId, totalRows, totalMsgs, + (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime, + currentLenOfMsg*1000.0/(1024*1024)/deltaTime); + break; + } + } + + pInfo->consumeMsgCnt = totalMsgs; + pInfo->consumeRowCnt = totalRows; + pInfo->consumeLen = totalLenOfMsg; + +} + + +void* ombConsumeThreadFunc(void* param) { + SThreadInfo* pInfo = (SThreadInfo*)param; + + //################### set key ######################## + tmq_conf_t* conf = tmq_conf_new(); + // tmq_conf_set(conf, "td.connect.ip", "localhost"); + // tmq_conf_set(conf, "td.connect.port", "6030"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + // tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, pInfo); + tmq_conf_set(conf, "group.id", "ombCgrp"); + // tmq_conf_set(conf, "msg.with.table.name", "true"); + // tmq_conf_set(conf, "client.id", "c-001"); + // tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "enable.auto.commit", "false"); + // tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); + // tmq_conf_set(conf, "auto.offset.reset", "none"); + // tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + // + if (g_stConfInfo.useSnapshot) { + tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + } + + pInfo->tmq = tmq_consumer_new(conf, NULL, 0); + + tmq_conf_destroy(conf); + + //################### set topic ########################## + pInfo->topicList = tmq_list_new(); + tmq_list_append(pInfo->topicList, g_stConfInfo.topic); + + if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { + taosFprintfFile(g_fp, "create consumer fail! tmq is null or topicList is null\n"); + return NULL; + } + + int32_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); + if (err != 0) { + pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_subscribe() fail! reason: %s\n", tmq_err2str(err)); + return NULL; + } + + tmq_list_destroy(pInfo->topicList); + pInfo->topicList = NULL; + + omb_loop_consume(pInfo); + + err = tmq_unsubscribe(pInfo->tmq); + if (err != 0) { + pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_unsubscribe()! reason: %s\n", tmq_err2str(err)); + } + + err = tmq_consumer_close(pInfo->tmq); + if (err != 0) { + pError("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_consumer_close()! reason: %s\n", tmq_err2str(err)); + } + pInfo->tmq = NULL; + + return NULL; +} + + + +static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type) { + TAOS_RES *res = taos_query(taos, command); + int32_t code = taos_errno(res); + + if (code != 0) { + pPrint("%s Failed to execute <%s>, reason: %s %s", GREEN, command, taos_errstr(res), NC); + taos_free_result(res); + return -1; + } + + if (INSERT_TYPE == type) { + int affectedRows = taos_affected_rows(res); + taos_free_result(res); + return affectedRows; + } + + taos_free_result(res); + return 0; +} + +void* ombProduceThreadFunc(void* param) { + SThreadInfo* pInfo = (SThreadInfo*)param; + + pInfo->taos = createNewTaosConnect(); + if (pInfo->taos == NULL) { + taosFprintfFile(g_fp, "taos_connect() fail, can not start producers!\n"); + return NULL; + } + + int64_t affectedRowsTotal = 0; + int64_t sendMsgs = 0; + + uint32_t totalSendLoopTimes = g_stConfInfo.runDurationMinutes * 60 * 1000 / SEND_TIME_UNIT; // send some msgs per 10ms + uint32_t batchPerTblTimes = pInfo->producerRate / 100 / g_stConfInfo.batchSize; + uint32_t remainder = (pInfo->producerRate / 100) % g_stConfInfo.batchSize; + if (remainder) { + batchPerTblTimes += 1; + } + + char* sqlBuf = taosMemoryMalloc(MAX_SQL_LEN); + if (NULL == sqlBuf) { + printf("malloc fail for sqlBuf\n"); + taos_close(pInfo->taos); + pInfo->taos = NULL; + return NULL; + } + + printf("Produce Info: totalSendLoopTimes: %d, batchPerTblTimes: %d, producerRate: %d\n", totalSendLoopTimes, batchPerTblTimes, pInfo->producerRate); + + char ctbName[64] = {0}; + sprintf(ctbName, "%s.ctb%d", g_stConfInfo.dbName, pInfo->consumerId); + + int64_t lastPrintTime = taosGetTimestampUs(); + int64_t totalMsgLen = 0; + //int64_t timeStamp = taosGetTimestampUs(); + while (totalSendLoopTimes) { + int64_t startTs = taosGetTimestampUs(); + for (int i = 0; i < batchPerTblTimes; ++i) { + uint32_t msgsOfSql = g_stConfInfo.batchSize; + if ((i == batchPerTblTimes - 1) && (0 != remainder)) { + msgsOfSql = remainder; + } + int len = 0; + len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "insert into %s values ", ctbName); + for (int j = 0; j < msgsOfSql; j++) { + int64_t timeStamp = taosGetTimestampNs(); + len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "(%" PRId64 ", \"%s\")", timeStamp, g_payload); + sendMsgs++; + pInfo->totalProduceMsgs++; + } + + totalMsgLen += len; + pInfo->totalMsgsLen += len; + + int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE); + if (affectedRows < 0) { + taos_close(pInfo->taos); + pInfo->taos = NULL; + return NULL; + } + + affectedRowsTotal += affectedRows; + + //printf("Produce Info: affectedRows: %" PRId64 "\n", affectedRows); + } + totalSendLoopTimes -= 1; + + // calc spent time + int64_t currentTs = taosGetTimestampUs(); + int64_t delta = currentTs - startTs; + if (delta < SEND_TIME_UNIT * 1000) { + int64_t sleepLen = (int32_t)(SEND_TIME_UNIT * 1000 - delta); + //printf("sleep %" PRId64 " us, use time: %" PRId64 " us\n", sleepLen, delta); + taosUsleep((int32_t)sleepLen); + } + + currentTs = taosGetTimestampUs(); + delta = currentTs - lastPrintTime; + if (delta > 10 * 1000 * 1000) { + printf("producer[%d] info: %" PRId64 " msgs, %" PRId64 " Byte, %" PRId64 " us, totalSendLoopTimes: %d\n", + pInfo->consumerId, sendMsgs, totalMsgLen, delta, totalSendLoopTimes); + printf("producer[%d] rate: %1.f msgs/s, %1.f KB/s\n", + pInfo->consumerId, + sendMsgs * 1000.0 * 1000 / delta, + (totalMsgLen / 1024.0) / (delta / (1000*1000))); + lastPrintTime = currentTs; + sendMsgs = 0; + totalMsgLen = 0; + } + } + + printf("affectedRowsTotal: %"PRId64"\n", affectedRowsTotal); + taos_close(pInfo->taos); + pInfo->taos = NULL; + return NULL; +} + + +void printProduceInfo(int64_t start) { + int64_t totalMsgs = 0; + int64_t totalLenOfMsgs = 0; + for (int i = 0; i < g_stConfInfo.producers; i++) { + totalMsgs += g_stConfInfo.stProdThreads[i].totalProduceMsgs; + totalLenOfMsgs += g_stConfInfo.stProdThreads[i].totalMsgsLen; + } + + int64_t end = taosGetTimestampUs(); + + int64_t t = end - start; + if (0 == t) t = 1; + + double tInMs = (double)t / 1000000.0; + printf("Spent %.3f seconds to prod %" PRIu64 " msgs, %" PRIu64 " Byte\n\n", tInMs, totalMsgs, totalLenOfMsgs); + + + printf("Spent %.3f seconds to prod %" PRIu64 " msgs with %d producer(s), throughput: %.3f msgs/s, %.1f MB/s\n\n", + tInMs, totalMsgs, g_stConfInfo.producers, + (double)totalMsgs / tInMs, + (double)totalLenOfMsgs/(1024.0*1024)/tInMs); + return; +} + + +void startOmbConsume() { + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + + if (0 != g_stConfInfo.producers) { + TAOS* taos = createNewTaosConnect(); + if (taos == NULL) { + taosFprintfFile(g_fp, "taos_connect() fail, can not create db, stbl, ctbl, topic!\n"); + return ; + } + + char stbName[16] = "stb"; + char ctbPrefix[16] = "ctb"; + + char sql[256] = {0}; + sprintf(sql, "drop database if exists %s", g_stConfInfo.dbName); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + + sprintf(sql, "create database if not exists %s precision 'ns' vgroups %d", g_stConfInfo.dbName, g_stConfInfo.producers); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + + sprintf(sql, "create stable %s.%s (ts timestamp, payload binary(%d)) tags (t bigint) ", g_stConfInfo.dbName, stbName, g_stConfInfo.payloadLen); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + + for (int i = 0; i < g_stConfInfo.producers; i++) { + sprintf(sql, "create table %s.%s%d using %s.stb tags(%d) ", g_stConfInfo.dbName, ctbPrefix, i, g_stConfInfo.dbName, i); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + } + + // create topic + sprintf(sql, "create topic %s as stable %s.%s", g_stConfInfo.topic, g_stConfInfo.dbName, stbName); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + + + int32_t producerRate = ceil(g_stConfInfo.producerRate / g_stConfInfo.producers); + + printf("==== create %d produce thread ====\n", g_stConfInfo.producers); + for (int32_t i = 0; i < g_stConfInfo.producers; ++i) { + g_stConfInfo.stProdThreads[i].consumerId = i; + g_stConfInfo.stProdThreads[i].producerRate = producerRate; + taosThreadCreate(&(g_stConfInfo.stProdThreads[i].thread), &thattr, ombProduceThreadFunc, + (void*)(&(g_stConfInfo.stProdThreads[i]))); + } + + if (0 == g_stConfInfo.numOfThread) { + int64_t start = taosGetTimestampUs(); + for (int32_t i = 0; i < g_stConfInfo.producers; i++) { + taosThreadJoin(g_stConfInfo.stProdThreads[i].thread, NULL); + taosThreadClear(&g_stConfInfo.stProdThreads[i].thread); + } + + printProduceInfo(start); + + taosFprintfFile(g_fp, "==== close tmqlog ====\n"); + taosCloseFile(&g_fp); + taos_close(taos); + return; + } + + taos_close(taos); + } + + // pthread_create one thread to consume + taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread); + for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) { + g_stConfInfo.stThreads[i].consumerId = i; + taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, ombConsumeThreadFunc, + (void*)(&(g_stConfInfo.stThreads[i]))); + } + + int64_t start = taosGetTimestampUs(); + + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL); + taosThreadClear(&g_stConfInfo.stThreads[i].thread); + } + + int64_t end = taosGetTimestampUs(); + + int64_t totalRows = 0; + int64_t totalMsgs = 0; + int64_t totalLenOfMsgs = 0; + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + totalMsgs += g_stConfInfo.stThreads[i].consumeMsgCnt; + totalLenOfMsgs += g_stConfInfo.stThreads[i].consumeLen; + totalRows += g_stConfInfo.stThreads[i].consumeRowCnt; + } + + int64_t t = end - start; + if (0 == t) t = 1; + + double tInMs = (double)t / 1000000.0; + taosFprintfFile(g_fp, + "Spent %.3f seconds to poll msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/s, %.1f MB/s\n\n", + tInMs, totalMsgs, g_stConfInfo.numOfThread, + (double)(totalMsgs / tInMs), + (double)totalLenOfMsgs/(1024*1024)/tInMs); + + printf("Spent %.3f seconds to cons rows: %" PRIu64 " msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/s, %.1f MB/s\n\n", + tInMs, totalRows, totalMsgs, g_stConfInfo.numOfThread, + (double)(totalMsgs / tInMs), + (double)totalLenOfMsgs/(1024*1024)/tInMs); + + taosFprintfFile(g_fp, "==== close tmqlog ====\n"); + taosCloseFile(&g_fp); + + return; +} + + +int main(int32_t argc, char* argv[]) { + parseArgument(argc, argv); + + if (0 != strlen(g_stConfInfo.topic)) { + startOmbConsume(); + return 0; + } + + int32_t retCode = getConsumeInfo(); + if (0 != retCode) { + return -1; + } + + saveConfigToLogFile(); + + tmqSetSignalHandle(); + + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + + // pthread_create one thread to consume + taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread); + for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) { + taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc, + (void*)(&(g_stConfInfo.stThreads[i]))); + } + + int64_t start = taosGetTimestampUs(); + + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL); + taosThreadClear(&g_stConfInfo.stThreads[i].thread); + } + + int64_t end = taosGetTimestampUs(); + + int64_t totalMsgs = 0; + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + totalMsgs += g_stConfInfo.stThreads[i].consumeMsgCnt; + } + + int64_t t = end - start; + if (0 == t) t = 1; + + double tInMs = (double)t / 1000000.0; + taosFprintfFile(g_fp, + "Spent %.3f seconds to poll msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/second\n\n", + tInMs, totalMsgs, g_stConfInfo.numOfThread, (double)(totalMsgs / tInMs)); + + taosFprintfFile(g_fp, "==== close tmqlog ====\n"); + taosCloseFile(&g_fp); + + return 0; +} diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c new file mode 100644 index 0000000000..f917b9159e --- /dev/null +++ b/utils/test/c/tmq_taosx_ci.c @@ -0,0 +1,623 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "taos.h" +#include "types.h" + +static int running = 1; +TdFilePtr g_fp = NULL; +typedef struct{ + bool snapShot; + bool dropTable; + bool subTable; + int srcVgroups; + int dstVgroups; + char dir[64]; +}Config; + +Config g_conf = {0}; + +static TAOS* use_db(){ + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return NULL; + } + + TAOS_RES* pRes = taos_query(pConn, "use db_taosx"); + if (taos_errno(pRes) != 0) { + printf("error in use db_taosx, reason:%s\n", taos_errstr(pRes)); + return NULL; + } + taos_free_result(pRes); + return pConn; +} + +static void msg_process(TAOS_RES* msg) { + printf("-----------topic-------------: %s\n", tmq_get_topic_name(msg)); + printf("db: %s\n", tmq_get_db_name(msg)); + printf("vg: %d\n", tmq_get_vgroup_id(msg)); + TAOS *pConn = use_db(); + if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { + char* result = tmq_get_json_meta(msg); + if (result) { + printf("meta result: %s\n", result); + } + if(g_fp){ + taosFprintfFile(g_fp, result); + taosFprintfFile(g_fp, "\n"); + } + + tmq_free_json_meta(result); + } + + tmq_raw_data raw = {0}; + tmq_get_raw(msg, &raw); + int32_t ret = tmq_write_raw(pConn, raw); + printf("write raw data: %s\n", tmq_err2str(ret)); + + taos_close(pConn); +} + +int buildDatabase(TAOS* pConn, TAOS_RES* pRes){ + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000, \"ttt\", true)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct0 values(1626006833400, 1, 2, 'a')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct1 using st1(t1) tags(2000)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table ct1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct2 using st1(t1) tags(NULL)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table ct2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct1 values(1626006833600, 3, 4, 'b')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct3 using st1(t1) tags(3000)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct3 values(1626006833600, 5, 6, 'c') ct1 values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, 'ddd') ct0 values(1626006833603, 4, 3, 'hwj') ct1 values(now+5s, 23, 32, 's21ds')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table st1 add column c4 bigint"); + if (taos_errno(pRes) != 0) { + printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table st1 modify column c3 binary(64)"); + if (taos_errno(pRes) != 0) { + printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct3 values(1626006833605, 53, 63, 'cffffffffffffffffffffffffffff', 8989898899999) (1626006833609, 51, 62, 'c333', 940)"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct3 select * from ct1"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table st1 add tag t2 binary(64)"); + if (taos_errno(pRes) != 0) { + printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table ct3 set tag t1=5000"); + if (taos_errno(pRes) != 0) { + printf("failed to slter child table ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "delete from abc1 .ct3 where ts < 1626006833606"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + if(g_conf.dropTable){ + pRes = taos_query(pConn, "drop table ct3 ct1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop child table ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop table st1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + } + + pRes = taos_query(pConn, "create table if not exists n1(ts timestamp, c1 int, c2 nchar(4))"); + if (taos_errno(pRes) != 0) { + printf("failed to create normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 add column c3 bigint"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 modify column c2 nchar(8)"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 rename column c3 cc3"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 comment 'hello'"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 drop column c1"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into n1 values(now, 'eeee', 8989898899999) (now+9s, 'c333', 940)"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + if(g_conf.dropTable){ + pRes = taos_query(pConn, "drop table n1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + } + + pRes = taos_query(pConn, "create table jt(ts timestamp, i int) tags(t json)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table jt1 using jt tags('{\"k1\":1, \"k2\":\"hello\"}')"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table jt2 using jt tags('')"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into jt1 values(now, 1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into jt2 values(now, 11)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + if(g_conf.dropTable){ + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop table st1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + } + return 0; +} + +int buildStable(TAOS* pConn, TAOS_RES* pRes){ + pRes = taos_query(pConn, "CREATE STABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` VARCHAR(16))"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table meters, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d0 using meters tags(1, 'San Francisco')"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table d0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d1 using meters tags(2, 'Beijing')"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table d1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stream meters_summary_s into meters_summary as select _wstart, max(current) as current, groupid, location from meters partition by groupid, location interval(10m)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table meters_summary, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into d0 (ts, current) values (now, 120)"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into table d0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + return 0; +} + +int32_t init_env() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_taosx"); + if (taos_errno(pRes) != 0) { + printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + char sql[128] = {0}; + snprintf(sql, 128, "create database if not exists db_taosx vgroups %d", g_conf.dstVgroups); + pRes = taos_query(pConn, sql); + if (taos_errno(pRes) != 0) { + printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop topic if exists topic_db"); + if (taos_errno(pRes) != 0) { + printf("error in drop topic, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop topic if exists meters_summary_t1"); + if (taos_errno(pRes) != 0) { + printf("error in drop topic, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists abc1"); + if (taos_errno(pRes) != 0) { + printf("error in drop db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + snprintf(sql, 128, "create database if not exists abc1 vgroups %d", g_conf.srcVgroups); + pRes = taos_query(pConn, sql); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + if(g_conf.subTable){ + buildStable(pConn, pRes); + }else{ + buildDatabase(pConn, pRes); + } + + taos_close(pConn); + return 0; +} + +int32_t create_topic() { + printf("create topic\n"); + TAOS_RES* pRes; + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + if(g_conf.subTable){ + pRes = taos_query(pConn, "create topic meters_summary_t1 with meta as stable meters_summary"); + if (taos_errno(pRes) != 0) { + printf("failed to create topic meters_summary_t1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + }else{ + pRes = taos_query(pConn, "create topic topic_db with meta as database abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to create topic topic_db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + } + + taos_close(pConn); + return 0; +} + +void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + printf("commit %d tmq %p param %p\n", code, tmq, param); +} + +tmq_t* build_consumer() { + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set(conf, "client.id", "my app 1"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "enable.heartbeat.background", "true"); + + if(g_conf.snapShot){ + tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + } + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + assert(tmq); + tmq_conf_destroy(conf); + return tmq; +} + +tmq_list_t* build_topic_list() { + tmq_list_t* topic_list = tmq_list_new(); + if(g_conf.subTable){ + tmq_list_append(topic_list, "meters_summary_t1"); + }else{ + tmq_list_append(topic_list, "topic_db"); + } + return topic_list; +} + +void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { + int32_t code; + + if ((code = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); + printf("subscribe err\n"); + return; + } + int32_t cnt = 0; + while (running) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000); + if (tmqmessage) { + cnt++; + msg_process(tmqmessage); + taos_free_result(tmqmessage); + }else{ + break; + } + } + + code = tmq_consumer_close(tmq); + if (code) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +void initLogFile() { + char f1[256] = {0}; + char f2[256] = {0}; + + if(g_conf.snapShot){ + sprintf(f1, "%s/../log/tmq_taosx_tmp_snapshot.source", g_conf.dir); + sprintf(f2, "%s/../log/tmq_taosx_tmp_snapshot.result", g_conf.dir); + }else{ + sprintf(f1, "%s/../log/tmq_taosx_tmp.source", g_conf.dir); + sprintf(f2, "%s/../log/tmq_taosx_tmp.result", g_conf.dir); + } + + TdFilePtr pFile = taosOpenFile(f1, TD_FILE_TEXT | TD_FILE_TRUNC | TD_FILE_STREAM); + if (NULL == pFile) { + fprintf(stderr, "Failed to open %s for save result\n", f1); + exit(-1); + } + g_fp = pFile; + + TdFilePtr pFile2 = taosOpenFile(f2, TD_FILE_TEXT | TD_FILE_TRUNC | TD_FILE_STREAM); + if (NULL == pFile2) { + fprintf(stderr, "Failed to open %s for save result\n", f2); + exit(-1); + } + + if(g_conf.snapShot){ + char *result[] = { + "{\"type\":\"create\",\"tableName\":\"st1\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":64},{\"name\":\"c4\",\"type\":5}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1},{\"name\":\"t2\",\"type\":8,\"length\":64}]}", + "{\"type\":\"create\",\"tableName\":\"ct0\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}", + "{\"type\":\"create\",\"tableName\":\"ct1\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}]}", + "{\"type\":\"create\",\"tableName\":\"ct2\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[]}", + "{\"type\":\"create\",\"tableName\":\"ct3\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":5000}]}", + "{\"type\":\"create\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c2\",\"type\":10,\"length\":8},{\"name\":\"cc3\",\"type\":5}],\"tags\":[]}", + "{\"type\":\"create\",\"tableName\":\"jt\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", + "{\"type\":\"create\",\"tableName\":\"jt1\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}]}", + "{\"type\":\"create\",\"tableName\":\"jt2\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]}", + }; + + for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){ + taosFprintfFile(pFile2, result[i]); + taosFprintfFile(pFile2, "\n"); + } + }else{ + char *result[] = { + "{\"type\":\"create\",\"tableName\":\"st1\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", + "{\"type\":\"create\",\"tableName\":\"ct0\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}", + "{\"type\":\"create\",\"tableName\":\"ct1\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}]}", + "{\"type\":\"create\",\"tableName\":\"ct2\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[]}", + "{\"type\":\"create\",\"tableName\":\"ct3\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":3000}]}", + "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":5,\"colName\":\"c4\",\"colType\":5}", + "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":7,\"colName\":\"c3\",\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":1,\"colName\":\"t2\",\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableName\":\"ct3\",\"tableType\":\"child\",\"alterType\":4,\"colName\":\"t1\",\"colValue\":\"5000\",\"colValueNull\":false}", + "{\"type\":\"create\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}", + "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":5,\"colName\":\"c3\",\"colType\":5}", + "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":7,\"colName\":\"c2\",\"colType\":10,\"colLength\":8}", + "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":10,\"colName\":\"c3\",\"colNewName\":\"cc3\"}", + "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":9}", + "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":6,\"colName\":\"c1\"}", + "{\"type\":\"create\",\"tableName\":\"jt\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", + "{\"type\":\"create\",\"tableName\":\"jt1\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}]}", + "{\"type\":\"create\",\"tableName\":\"jt2\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]}" + }; + + for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){ + taosFprintfFile(pFile2, result[i]); + taosFprintfFile(pFile2, "\n"); + } + } + + taosCloseFile(&pFile2); +} + +int main(int argc, char* argv[]) { + for (int32_t i = 1; i < argc; i++) { + if(strcmp(argv[i], "-c") == 0){ + strcpy(g_conf.dir, argv[++i]); + }else if(strcmp(argv[i], "-s") == 0){ + g_conf.snapShot = true; + }else if(strcmp(argv[i], "-d") == 0){ + g_conf.dropTable = true; + }else if(strcmp(argv[i], "-sv") == 0){ + g_conf.srcVgroups = atol(argv[++i]); + }else if(strcmp(argv[i], "-dv") == 0){ + g_conf.dstVgroups = atol(argv[++i]); + }else if(strcmp(argv[i], "-t") == 0){ + g_conf.subTable = true; + } + } + + printf("env init\n"); + if(strlen(g_conf.dir) != 0){ + initLogFile(); + } + + if (init_env() < 0) { + return -1; + } + create_topic(); + + tmq_t* tmq = build_consumer(); + tmq_list_t* topic_list = build_topic_list(); + basic_consume_loop(tmq, topic_list); + taosCloseFile(&g_fp); +} diff --git a/utils/tsim/CMakeLists.txt b/utils/tsim/CMakeLists.txt new file mode 100644 index 0000000000..c2cf7ac3c5 --- /dev/null +++ b/utils/tsim/CMakeLists.txt @@ -0,0 +1,14 @@ +aux_source_directory(src TSIM_SRC) +add_executable(tsim ${TSIM_SRC}) +target_link_libraries( + tsim + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os + PUBLIC cjson +) +target_include_directories( + tsim + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) diff --git a/utils/tsim/inc/simInt.h b/utils/tsim/inc/simInt.h new file mode 100644 index 0000000000..f9e96fc67b --- /dev/null +++ b/utils/tsim/inc/simInt.h @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_SIM_INT_H_ +#define _TD_SIM_INT_H_ + +#include "os.h" + +#include "cJSON.h" +#include "tconfig.h" +#include "taos.h" +#include "taoserror.h" +#include "tidpool.h" +#include "tlog.h" +#include "ttimer.h" +#include "ttypes.h" +#include "tutil.h" +#include "tglobal.h" + +#define MAX_MAIN_SCRIPT_NUM 10 +#define MAX_BACKGROUND_SCRIPT_NUM 10 +#define MAX_FILE_NAME_LEN 256 +#define MAX_ERROR_LEN 1024 +#define MAX_QUERY_VALUE_LEN 1024 +#define MAX_QUERY_COL_NUM 100 +#define MAX_QUERY_ROW_NUM 100 +#define MAX_SYSTEM_RESULT_LEN 2048 +#define MAX_VAR_LEN 100 +#define MAX_VAR_NAME_LEN 32 +#define MAX_VAR_VAL_LEN 80 +#define MAX_OPT_NAME_LEN 32 +#define MAX_SIM_CMD_NAME_LEN 40 + +#ifdef LINUX +#define SUCCESS_PREFIX "\033[44;32;1m" +#define SUCCESS_POSTFIX "\033[0m" +#define FAILED_PREFIX "\033[44;31;1m" +#define FAILED_POSTFIX "\033[0m" +#else +#define SUCCESS_PREFIX "" +#define SUCCESS_POSTFIX "" +#define FAILED_PREFIX "" +#define FAILED_POSTFIX "" +#endif + +#define simFatal(...) { if (simDebugFlag & DEBUG_FATAL) { taosPrintLog("SIM FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define simError(...) { if (simDebugFlag & DEBUG_ERROR) { taosPrintLog("SIM ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define simWarn(...) { if (simDebugFlag & DEBUG_WARN) { taosPrintLog("SIM WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define simInfo(...) { if (simDebugFlag & DEBUG_INFO) { taosPrintLog("SIM ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define simDebug(...) { if (simDebugFlag & DEBUG_DEBUG) { taosPrintLog("SIM ", DEBUG_DEBUG, simDebugFlag, __VA_ARGS__); }} +#define simTrace(...) { if (simDebugFlag & DEBUG_TRACE) { taosPrintLog("SIM ", DEBUG_TRACE, simDebugFlag, __VA_ARGS__); }} + +enum { SIM_SCRIPT_TYPE_MAIN, SIM_SCRIPT_TYPE_BACKGROUND }; + +enum { + SIM_CMD_EXP, + SIM_CMD_IF, + SIM_CMD_ELIF, + SIM_CMD_ELSE, + SIM_CMD_ENDI, + SIM_CMD_WHILE, + SIM_CMD_ENDW, + SIM_CMD_SWITCH, + SIM_CMD_CASE, + SIM_CMD_DEFAULT, + SIM_CMD_CONTINUE, + SIM_CMD_BREAK, + SIM_CMD_ENDS, + SIM_CMD_SLEEP, + SIM_CMD_GOTO, + SIM_CMD_RUN, + SIM_CMD_RUN_BACK, + SIM_CMD_PRINT, + SIM_CMD_SYSTEM, + SIM_CMD_SYSTEM_CONTENT, + SIM_CMD_SQL, + SIM_CMD_SQL_ERROR, + SIM_CMD_SQL_SLOW, + SIM_CMD_RESTFUL, + SIM_CMD_TEST, + SIM_CMD_RETURN, + SIM_CMD_LINE_INSERT, + SIM_CMD_LINE_INSERT_ERROR, + SIM_CMD_END +}; + +enum { + SQL_JUMP_FALSE, + SQL_JUMP_TRUE, +}; + +struct _script_t; +typedef struct _cmd_t { + int16_t cmdno; + int16_t nlen; + char name[MAX_SIM_CMD_NAME_LEN]; + bool (*parseCmd)(char *, struct _cmd_t *, int32_t); + bool (*executeCmd)(struct _script_t *script, char *option); + struct _cmd_t *next; +} SCommand; + +typedef struct { + int16_t cmdno; + int16_t jump; // jump position + int16_t errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag + // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */ + int16_t lineNum; // correspodning line number in original file + int32_t optionOffset; // relative option offset +} SCmdLine; + +typedef struct _var_t { + char varName[MAX_VAR_NAME_LEN]; + char varValue[MAX_VAR_VAL_LEN]; + char varNameLen; +} SVariable; + +typedef struct _script_t { + int32_t type; + bool killed; + void *taos; + char rows[12]; // number of rows data retrieved + char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN]; // query results + char system_exit_code[12]; + char system_ret_content[MAX_SYSTEM_RESULT_LEN]; + int32_t varLen; + int32_t linePos; // current cmd position + int32_t numOfLines; // number of lines in the script + int32_t bgScriptLen; + char fileName[MAX_FILE_NAME_LEN]; // script file name + char error[MAX_ERROR_LEN]; + char *optionBuffer; + SCmdLine *lines; // command list + SVariable variables[MAX_VAR_LEN]; + TdThread bgPid; + char auth[128]; + struct _script_t *bgScripts[MAX_BACKGROUND_SCRIPT_NUM]; +} SScript; + +extern SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; +extern SCommand simCmdList[]; +extern int32_t simScriptPos; +extern int32_t simScriptSucced; +extern int32_t simDebugFlag; +extern char simScriptDir[]; +extern bool abortExecution; +extern bool useMultiProcess; +extern bool useValgrind; + +SScript *simParseScript(char *fileName); +SScript *simProcessCallOver(SScript *script); +void *simExecuteScript(void *script); +void simInitsimCmdList(); +bool simSystemInit(); +void simSystemCleanUp(); +char *simGetVariable(SScript *script, char *varName, int32_t varLen); +bool simExecuteExpCmd(SScript *script, char *option); +bool simExecuteTestCmd(SScript *script, char *option); +bool simExecuteGotoCmd(SScript *script, char *option); +bool simExecuteRunCmd(SScript *script, char *option); +bool simExecuteRunBackCmd(SScript *script, char *option); +bool simExecuteSystemCmd(SScript *script, char *option); +bool simExecuteSystemContentCmd(SScript *script, char *option); +bool simExecutePrintCmd(SScript *script, char *option); +bool simExecuteSleepCmd(SScript *script, char *option); +bool simExecuteReturnCmd(SScript *script, char *option); +bool simExecuteSqlCmd(SScript *script, char *option); +bool simExecuteSqlErrorCmd(SScript *script, char *rest); +bool simExecuteSqlSlowCmd(SScript *script, char *option); +bool simExecuteRestfulCmd(SScript *script, char *rest); +bool simExecuteLineInsertCmd(SScript *script, char *option); +bool simExecuteLineInsertErrorCmd(SScript *script, char *option); +void simVisuallizeOption(SScript *script, char *src, char *dst); + +#endif /*_TD_SIM_INT_H_*/ \ No newline at end of file diff --git a/utils/tsim/inc/simParse.h b/utils/tsim/inc/simParse.h new file mode 100644 index 0000000000..2a683f9561 --- /dev/null +++ b/utils/tsim/inc/simParse.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_SIM_PARSE_H_ +#define _TD_SIM_PARSE_H_ + +#include "simInt.h" + +#define MAX_NUM_CMD 64 +#define MAX_NUM_LABLES 100 +#define MAX_LABEL_LEN 40 +#define MAX_NUM_BLOCK 100 +#define MAX_NUM_JUMP 100 +#define MAX_LINE_LEN 3000 +#define MAX_CMD_LINES 2048 +#define MAX_OPTION_BUFFER 64000 + +enum { + BLOCK_IF, + BLOCK_WHILE, + BLOCK_SWITCH, +}; + +/* label stack */ +typedef struct { + int8_t top; /* number of labels */ + int16_t pos[MAX_NUM_LABLES]; /* the position of the label */ + char label[MAX_NUM_LABLES][MAX_LABEL_LEN]; /* name of the label */ +} SLabel; + +/* block definition */ +typedef struct { + int8_t top; /* the number of blocks stacked */ + char type[MAX_NUM_BLOCK]; /* the block type */ + int16_t *pos[MAX_NUM_BLOCK]; /* position of the jump for if/elif/case */ + int16_t back[MAX_NUM_BLOCK]; /* go back, endw and continue */ + char numJump[MAX_NUM_BLOCK]; + int16_t *jump[MAX_NUM_BLOCK][MAX_NUM_JUMP]; /* break or elif */ + char sexp[MAX_NUM_BLOCK][40]; /*switch expression */ + char sexpLen[MAX_NUM_BLOCK]; /*switch expression length */ +} SBlock; + +bool simParseExpression(char *token, int32_t lineNum); + +#endif /*_TD_SIM_PARSE_H_*/ \ No newline at end of file diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExe.c new file mode 100644 index 0000000000..16732ff9a1 --- /dev/null +++ b/utils/tsim/src/simExe.c @@ -0,0 +1,1016 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "simInt.h" + +void simLogSql(char *sql, bool useSharp) { + static TdFilePtr pFile = NULL; + char filename[256]; + sprintf(filename, "%s/sim.sql", simScriptDir); + if (pFile == NULL) { + // fp = fopen(filename, "w"); + pFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + if (pFile == NULL) { + fprintf(stderr, "ERROR: failed to open file: %s\n", filename); + return; + } + } + if (useSharp) { + taosFprintfFile(pFile, "# %s;\n", sql); + } else { + taosFprintfFile(pFile, "%s;\n", sql); + } + + taosFsyncFile(pFile); +} + +char *simParseArbitratorName(char *varName) { + static char hostName[140]; +#ifdef WINDOWS + taosGetFqdn(hostName); + sprintf(&hostName[strlen(hostName)], ":%d", 8000); +#else + sprintf(hostName, "%s:%d", "localhost", 8000); +#endif + return hostName; +} + +char *simParseHostName(char *varName) { + static char hostName[140]; +#ifdef WINDOWS + hostName[0] = '\"'; + taosGetFqdn(&hostName[1]); + int strEndIndex = strlen(hostName); + hostName[strEndIndex] = '\"'; + hostName[strEndIndex + 1] = '\0'; +#else + sprintf(hostName, "%s", "localhost"); +#endif + return hostName; +} + +static void simFindFirstNum(const char *begin, int32_t beginLen, int32_t *num) { + if (beginLen <= 5) { + *num = 0; + } else { + *num = atoi(begin + 5); + } +} + +static void simFindSecondNum(const char *begin, int32_t beginLen, int32_t *num) { + const char *number = strstr(begin, "]["); + if (number == NULL) { + *num = 0; + } else { + *num = atoi(number + 2); + } +} + +static void simFindFirstKeyVal(const char *begin, int32_t beginLen, char *key, int32_t keyLen) { + key[0] = 0; + for (int32_t i = 5; i < beginLen && i - 5 < keyLen; ++i) { + if (begin[i] != 0 && begin[i] != ']' && begin[i] != ')') { + key[i - 5] = begin[i]; + } + } +} + +static void simFindSecondKeyNum(const char *begin, int32_t beginLen, int32_t *num) { + const char *number = strstr(begin, ")["); + if (number == NULL) { + *num = 0; + } else { + *num = atoi(number + 2); + } +} + +char *simGetVariable(SScript *script, char *varName, int32_t varLen) { + if (strncmp(varName, "hostname", 8) == 0) { + return simParseHostName(varName); + } + + if (strncmp(varName, "arbitrator", 10) == 0) { + return simParseArbitratorName(varName); + } + + if (strncmp(varName, "error", varLen) == 0) return script->error; + + if (strncmp(varName, "rows", varLen) == 0) return script->rows; + + if (strncmp(varName, "system_exit", varLen) == 0) return script->system_exit_code; + + if (strncmp(varName, "system_content", varLen) == 0) return script->system_ret_content; + + if (strncmp(varName, "data", 4) == 0) { + if (varLen < 6) { + return "null"; + } + + int32_t row = 0; + int32_t col = 0; + char keyVal[1024] = {0}; + int32_t keyLen = 1024; + + if (varName[4] == '[') { + // $data[0][1] + simFindFirstNum(varName, varLen, &row); + simFindSecondNum(varName, varLen, &col); + if (row < 0 || row >= MAX_QUERY_ROW_NUM) { + return "null"; + } + if (col < 0 || col >= MAX_QUERY_COL_NUM) { + return "null"; + } + simDebug("script:%s, data[%d][%d]=%s", script->fileName, row, col, script->data[row][col]); + return script->data[row][col]; + } else if (varName[4] == '(') { + // $data(db)[0] + simFindFirstKeyVal(varName, varLen, keyVal, keyLen); + simFindSecondKeyNum(varName, varLen, &col); + for (int32_t i = 0; i < MAX_QUERY_ROW_NUM; ++i) { + if (strncmp(keyVal, script->data[i][0], keyLen) == 0) { + simDebug("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]); + return script->data[i][col]; + } + } + } else if (varName[5] == '_') { + // data2_db + int32_t col = varName[4] - '0'; + if (col < 0 || col >= MAX_QUERY_COL_NUM) { + return "null"; + } + + char *keyName; + int32_t keyLen; + paGetToken(varName + 6, &keyName, &keyLen); + + for (int32_t i = 0; i < MAX_QUERY_ROW_NUM; ++i) { + if (strncmp(keyName, script->data[i][0], keyLen) == 0) { + simDebug("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]); + return script->data[i][col]; + } + } + return "null"; + } else if (varName[6] == '_') { + // data21_db + int32_t col = (varName[4] - '0') * 10 + (varName[5] - '0'); + if (col < 0 || col >= MAX_QUERY_COL_NUM) { + return "null"; + } + + char *keyName; + int32_t keyLen; + paGetToken(varName + 7, &keyName, &keyLen); + + for (int32_t i = 0; i < MAX_QUERY_ROW_NUM; ++i) { + if (strncmp(keyName, script->data[i][0], keyLen) == 0) { + simTrace("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]); + return script->data[i][col]; + } + } + return "null"; + } else { + // $data00 + int32_t row = varName[4] - '0'; + int32_t col = varName[5] - '0'; + if (row < 0 || row >= MAX_QUERY_ROW_NUM) { + return "null"; + } + if (col < 0 || col >= MAX_QUERY_COL_NUM) { + return "null"; + } + + simDebug("script:%s, data[%d][%d]=%s", script->fileName, row, col, script->data[row][col]); + return script->data[row][col]; + } + + return "null"; + } + + for (int32_t i = 0; i < script->varLen; ++i) { + SVariable *var = &script->variables[i]; + if (var->varNameLen != varLen) { + continue; + } + if (strncmp(varName, var->varName, varLen) == 0) { + return var->varValue; + } + } + + if (script->varLen >= MAX_VAR_LEN) { + simError("script:%s, too many varialbes:%d", script->fileName, script->varLen); + exit(0); + } + + SVariable *var = &script->variables[script->varLen]; + script->varLen++; + strncpy(var->varName, varName, varLen); + var->varNameLen = varLen; + var->varValue[0] = 0; + return var->varValue; +} + +int32_t simExecuteExpression(SScript *script, char *exp) { + char *op1, *op2, *var1, *var2, *var3, *rest; + int32_t op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1; + char t0[1024], t1[1024], t2[1024], t3[2048]; + int32_t result; + + rest = paGetToken(exp, &var1, &var1Len); + rest = paGetToken(rest, &op1, &op1Len); + rest = paGetToken(rest, &var2, &var2Len); + rest = paGetToken(rest, &op2, &op2Len); + + if (var1[0] == '$') + strcpy(t0, simGetVariable(script, var1 + 1, var1Len - 1)); + else { + memcpy(t0, var1, var1Len); + t0[var1Len] = 0; + } + + if (var2[0] == '$') + strcpy(t1, simGetVariable(script, var2 + 1, var2Len - 1)); + else { + memcpy(t1, var2, var2Len); + t1[var2Len] = 0; + } + + if (op2Len != 0) { + rest = paGetToken(rest, &var3, &var3Len); + + if (var3[0] == '$') + strcpy(t2, simGetVariable(script, var3 + 1, var3Len - 1)); + else { + memcpy(t2, var3, var3Len); + t2[var3Len] = 0; + } + + if (op2[0] == '+') { + sprintf(t3, "%lld", atoll(t1) + atoll(t2)); + } else if (op2[0] == '-') { + sprintf(t3, "%lld", atoll(t1) - atoll(t2)); + } else if (op2[0] == '*') { + sprintf(t3, "%lld", atoll(t1) * atoll(t2)); + } else if (op2[0] == '/') { + sprintf(t3, "%lld", atoll(t1) / atoll(t2)); + } else if (op2[0] == '.') { + sprintf(t3, "%s%s", t1, t2); + } + } else { + strcpy(t3, t1); + } + + result = 0; + + if (op1Len == 1) { + if (op1[0] == '=') { + strcpy(simGetVariable(script, var1 + 1, var1Len - 1), t3); + } else if (op1[0] == '<') { + val0 = atoi(t0); + val1 = atoi(t3); + if (val0 >= val1) result = -1; + } else if (op1[0] == '>') { + val0 = atoi(t0); + val1 = atoi(t3); + if (val0 <= val1) result = -1; + } + } else { + if (op1[0] == '=' && op1[1] == '=') { + if (strcmp(t0, t3) != 0) result = -1; + } else if (op1[0] == '!' && op1[1] == '=') { + if (strcmp(t0, t3) == 0) result = -1; + } else if (op1[0] == '<' && op1[1] == '=') { + val0 = atoi(t0); + val1 = atoi(t3); + if (val0 > val1) result = -1; + } else if (op1[0] == '>' && op1[1] == '=') { + val0 = atoi(t0); + val1 = atoi(t3); + if (val0 < val1) result = -1; + } + } + + return result; +} + +bool simExecuteExpCmd(SScript *script, char *option) { + simExecuteExpression(script, option); + script->linePos++; + return true; +} + +bool simExecuteTestCmd(SScript *script, char *option) { + int32_t result; + result = simExecuteExpression(script, option); + + if (result >= 0) + script->linePos++; + else + script->linePos = script->lines[script->linePos].jump; + + return true; +} + +bool simExecuteGotoCmd(SScript *script, char *option) { + script->linePos = script->lines[script->linePos].jump; + return true; +} + +bool simExecuteRunCmd(SScript *script, char *option) { + char *fileName = option; + if (fileName == NULL || strlen(fileName) == 0) { + sprintf(script->error, "lineNum:%d. script file is null", script->lines[script->linePos].lineNum); + return false; + } + + SScript *newScript = simParseScript(option); + if (newScript == NULL) { + sprintf(script->error, "lineNum:%d. parse file:%s error", script->lines[script->linePos].lineNum, fileName); + return false; + } + + simInfo("script:%s, start to execute", newScript->fileName); + + newScript->type = SIM_SCRIPT_TYPE_MAIN; + simScriptPos++; + simScriptList[simScriptPos] = newScript; + + script->linePos++; + return true; +} + +bool simExecuteRunBackCmd(SScript *script, char *option) { + char *fileName = option; + if (fileName == NULL || strlen(fileName) == 0) { + sprintf(script->error, "lineNum:%d. script file is null", script->lines[script->linePos].lineNum); + return false; + } + + SScript *newScript = simParseScript(option); + if (newScript == NULL) { + sprintf(script->error, "lineNum:%d. parse file:%s error", script->lines[script->linePos].lineNum, fileName); + return false; + } + + newScript->type = SIM_SCRIPT_TYPE_BACKGROUND; + script->bgScripts[script->bgScriptLen++] = newScript; + simInfo("script:%s, start to execute in background,", newScript->fileName); + + if (taosThreadCreate(&newScript->bgPid, NULL, simExecuteScript, (void *)newScript) != 0) { + sprintf(script->error, "lineNum:%d. create background thread failed", script->lines[script->linePos].lineNum); + return false; + } else { + simDebug("script:%s, background thread:0x%08" PRIx64 " is created", newScript->fileName, + taosGetPthreadId(newScript->bgPid)); + } + + script->linePos++; + return true; +} + +bool simReplaceStr(char *buf, char *src, char *dst) { + bool replaced = false; + char *begin = strstr(buf, src); + if (begin != NULL) { + int32_t srcLen = (int32_t)strlen(src); + int32_t dstLen = (int32_t)strlen(dst); + int32_t interval = (dstLen - srcLen); + int32_t remainLen = (int32_t)strlen(buf); + char *end = buf + remainLen; + *(end + interval) = 0; + + for (char *p = end; p >= begin; p--) { + *(p + interval) = *p; + } + + memcpy(begin, dst, dstLen); + replaced = true; + } + + simInfo("system cmd is %s", buf); + return replaced; +} + +bool simExecuteSystemCmd(SScript *script, char *option) { + char buf[4096] = {0}; + bool replaced = false; + +#ifndef WINDOWS + sprintf(buf, "cd %s; ", simScriptDir); + simVisuallizeOption(script, option, buf + strlen(buf)); +#else + sprintf(buf, "%s", simScriptDir); + simVisuallizeOption(script, option, buf + strlen(buf)); + simReplaceStr(buf, ".sh", ".bat"); +#endif + + if (useMultiProcess) { + simReplaceStr(buf, "deploy.sh", "deploy.sh -m"); + } + + if (useValgrind) { + replaced = simReplaceStr(buf, "exec.sh", "exec.sh -v"); + } + + simLogSql(buf, true); + int32_t code = system(buf); + int32_t repeatTimes = 0; + while (code < 0) { + simError("script:%s, failed to execute %s , code %d, errno:%d %s, repeatTimes:%d", script->fileName, buf, code, + errno, strerror(errno), repeatTimes); + taosMsleep(1000); + taosDflSignal(SIGCHLD); + if (repeatTimes++ >= 10) { + exit(0); + } + } + + sprintf(script->system_exit_code, "%d", code); + script->linePos++; + if (replaced && strstr(buf, "start") != NULL) { + simInfo("====> startup is slow in valgrind mode, so sleep 5 seconds after exec.sh -s start"); + taosMsleep(5000); + } + + return true; +} + +void simStoreSystemContentResult(SScript *script, char *filename) { + memset(script->system_ret_content, 0, MAX_SYSTEM_RESULT_LEN); + + TdFilePtr pFile; + // if ((fd = fopen(filename, "r")) != NULL) { + if ((pFile = taosOpenFile(filename, TD_FILE_READ)) != NULL) { + taosReadFile(pFile, script->system_ret_content, MAX_SYSTEM_RESULT_LEN - 1); + int32_t len = strlen(script->system_ret_content); + for (int32_t i = 0; i < len; ++i) { + if (script->system_ret_content[i] == '\n' || script->system_ret_content[i] == '\r') { + script->system_ret_content[i] = 0; + } + } + taosCloseFile(&pFile); + char rmCmd[MAX_FILE_NAME_LEN] = {0}; + sprintf(rmCmd, "rm -f %s", filename); +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-result" + system(rmCmd); +#pragma GCC diagnostic pop + } +} + +bool simExecuteSystemContentCmd(SScript *script, char *option) { + char buf[4096] = {0}; + char buf1[4096 + 512] = {0}; + char filename[400] = {0}; + sprintf(filename, "%s" TD_DIRSEP "%s.tmp", simScriptDir, script->fileName); + +#ifdef WINDOWS + sprintf(buf, "cd %s && ", simScriptDir); + simVisuallizeOption(script, option, buf + strlen(buf)); + sprintf(buf1, "%s > %s 2>nul", buf, filename); +#else + sprintf(buf, "cd %s; ", simScriptDir); + simVisuallizeOption(script, option, buf + strlen(buf)); + sprintf(buf1, "%s > %s 2>/dev/null", buf, filename); +#endif + + sprintf(script->system_exit_code, "%d", system(buf1)); + simStoreSystemContentResult(script, filename); + + script->linePos++; + return true; +} + +bool simExecutePrintCmd(SScript *script, char *rest) { + char buf[65536]; + + simVisuallizeOption(script, rest, buf); + rest = buf; + + simInfo("script:%s, %s", script->fileName, rest); + script->linePos++; + return true; +} + +bool simExecuteSleepCmd(SScript *script, char *option) { + int32_t delta; + char buf[1024]; + + simVisuallizeOption(script, option, buf); + option = buf; + + delta = atoi(option); + if (delta <= 0) delta = 5; + + simInfo("script:%s, sleep %dms begin", script->fileName, delta); + taosMsleep(delta); + simInfo("script:%s, sleep %dms finished", script->fileName, delta); + + char sleepStr[32] = {0}; + sprintf(sleepStr, "sleep %d", delta); + simLogSql(sleepStr, true); + + script->linePos++; + return true; +} + +bool simExecuteReturnCmd(SScript *script, char *option) { + char buf[1024]; + + simVisuallizeOption(script, option, buf); + option = buf; + + int32_t ret = 1; + if (option && option[0] != 0) ret = atoi(option); + + if (ret < 0) { + sprintf(script->error, "lineNum:%d. error return %s", script->lines[script->linePos].lineNum, option); + return false; + } else { + simInfo("script:%s, return cmd execute with:%d", script->fileName, ret); + script->linePos = script->numOfLines; + } + + script->linePos++; + return true; +} + +void simVisuallizeOption(SScript *script, char *src, char *dst) { + char *var, *token, *value; + int32_t dstLen, srcLen, tokenLen; + + dst[0] = 0, dstLen = 0; + + while (1) { + var = strchr(src, '$'); + if (var == NULL) break; + if (var && ((var - src - 1) > 0) && *(var - 1) == '\\') { + srcLen = (int32_t)(var - src - 1); + memcpy(dst + dstLen, src, srcLen); + dstLen += srcLen; + src = var; + break; + } + + srcLen = (int32_t)(var - src); + memcpy(dst + dstLen, src, srcLen); + dstLen += srcLen; + + src = paGetToken(var + 1, &token, &tokenLen); + value = simGetVariable(script, token, tokenLen); + + strcpy(dst + dstLen, value); + dstLen += (int32_t)strlen(value); + } + + strcpy(dst + dstLen, src); +} + +void simCloseNativeConnect(SScript *script) { + if (script->taos == NULL) return; + + simDebug("script:%s, taos:%p closed", script->fileName, script->taos); + taos_close(script->taos); + + script->taos = NULL; +} + +void simCloseTaosdConnect(SScript *script) { simCloseNativeConnect(script); } + +bool simCreateNativeConnect(SScript *script, char *user, char *pass) { + simCloseTaosdConnect(script); + void *taos = NULL; + for (int32_t attempt = 0; attempt < 10; ++attempt) { + if (abortExecution) { + script->killed = true; + return false; + } + + taos = taos_connect(NULL, user, pass, NULL, 0); + if (taos == NULL) { + simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), + attempt); + taosMsleep(1000); + } else { + simDebug("script:%s, user:%s connect taosd successed, attempt:%d", script->fileName, user, attempt); + break; + } + } + + if (taos == NULL) { + sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum, + taos_errstr(NULL)); + return false; + } + + script->taos = taos; + simDebug("script:%s, connect taosd successed, taos:%p", script->fileName, taos); + + return true; +} + +bool simCreateTaosdConnect(SScript *script, char *rest) { + char *user = TSDB_DEFAULT_USER; + char *token; + int32_t tokenLen; + rest = paGetToken(rest, &token, &tokenLen); + rest = paGetToken(rest, &token, &tokenLen); + if (tokenLen != 0) { + user = token; + } + + return simCreateNativeConnect(script, user, TSDB_DEFAULT_PASS); +} + +bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { + char timeStr[30] = {0}; + time_t tt; + struct tm tp; + SCmdLine *line = &script->lines[script->linePos]; + int32_t ret = -1; + + TAOS_RES *pSql = NULL; + + for (int32_t attempt = 0; attempt < 10; ++attempt) { + if (abortExecution) { + script->killed = true; + return false; + } + + simLogSql(rest, false); + pSql = taos_query(script->taos, rest); + ret = taos_errno(pSql); + + if (ret == TSDB_CODE_MND_STB_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) { + simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, + tstrerror(ret)); + ret = 0; + break; + } else if (ret != 0) { + simDebug("script:%s, taos:%p, %s failed, ret:%d:%s, error:%s", script->fileName, script->taos, rest, ret & 0XFFFF, + tstrerror(ret), taos_errstr(pSql)); + + if (line->errorJump == SQL_JUMP_TRUE) { + script->linePos = line->jump; + taos_free_result(pSql); + return true; + } + taosMsleep(1000); + } else { + break; + } + + taos_free_result(pSql); + } + + if (ret) { + sprintf(script->error, "lineNum:%d. sql:%s failed, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, tstrerror(ret)); + return false; + } + + int32_t numOfRows = 0; + int32_t num_fields = taos_field_count(pSql); + if (num_fields != 0) { + if (pSql == NULL) { + simDebug("script:%s, taos:%p, %s failed, result is null", script->fileName, script->taos, rest); + if (line->errorJump == SQL_JUMP_TRUE) { + script->linePos = line->jump; + return true; + } + + sprintf(script->error, "lineNum:%d. result set null, sql:%s", line->lineNum, rest); + return false; + } + + TAOS_ROW row; + + while ((row = taos_fetch_row(pSql))) { + if (numOfRows < MAX_QUERY_ROW_NUM) { + TAOS_FIELD *fields = taos_fetch_fields(pSql); + int32_t *length = taos_fetch_lengths(pSql); + + for (int32_t i = 0; i < num_fields; i++) { + char *value = NULL; + if (i < MAX_QUERY_COL_NUM) { + value = script->data[numOfRows][i]; + } + if (value == NULL) { + continue; + } + + if (row[i] == 0) { + strcpy(value, TSDB_DATA_NULL_STR); + continue; + } + + switch (fields[i].type) { + case TSDB_DATA_TYPE_BOOL: + sprintf(value, "%s", ((((int32_t)(*((char *)row[i]))) == 1) ? "1" : "0")); + break; + case TSDB_DATA_TYPE_TINYINT: + sprintf(value, "%d", *((int8_t *)row[i])); + break; + case TSDB_DATA_TYPE_UTINYINT: + sprintf(value, "%u", *((uint8_t *)row[i])); + break; + case TSDB_DATA_TYPE_SMALLINT: + sprintf(value, "%d", *((int16_t *)row[i])); + break; + case TSDB_DATA_TYPE_USMALLINT: + sprintf(value, "%u", *((uint16_t *)row[i])); + break; + case TSDB_DATA_TYPE_INT: + sprintf(value, "%d", *((int32_t *)row[i])); + break; + case TSDB_DATA_TYPE_UINT: + sprintf(value, "%u", *((uint32_t *)row[i])); + break; + case TSDB_DATA_TYPE_BIGINT: + sprintf(value, "%" PRId64, *((int64_t *)row[i])); + break; + case TSDB_DATA_TYPE_UBIGINT: + sprintf(value, "%" PRIu64, *((uint64_t *)row[i])); + break; + case TSDB_DATA_TYPE_FLOAT: + sprintf(value, "%.5f", GET_FLOAT_VAL(row[i])); + break; + case TSDB_DATA_TYPE_DOUBLE: + sprintf(value, "%.9lf", GET_DOUBLE_VAL(row[i])); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + if (length[i] < 0 || length[i] > 1 << 20) { + fprintf(stderr, "Invalid length(%d) of BINARY or NCHAR\n", length[i]); + exit(-1); + } + + memset(value, 0, MAX_QUERY_VALUE_LEN); + memcpy(value, row[i], length[i]); + value[length[i]] = 0; + // snprintf(value, fields[i].bytes, "%s", (char *)row[i]); + break; + case TSDB_DATA_TYPE_TIMESTAMP: { + int32_t precision = taos_result_precision(pSql); + if (precision == TSDB_TIME_PRECISION_MILLI) { + tt = (*(int64_t *)row[i]) / 1000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + tt = (*(int64_t *)row[i]) / 1000000; + } else { + tt = (*(int64_t *)row[i]) / 1000000000; + } + + taosLocalTime(&tt, &tp); + strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", &tp); + if (precision == TSDB_TIME_PRECISION_MILLI) { + sprintf(value, "%s.%03d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000)); + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + sprintf(value, "%s.%06d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000000)); + } else { + sprintf(value, "%s.%09d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000000000)); + } + + break; + } + default: + break; + } // end of switch + } // end of for + } // end of if + numOfRows++; + if (isSlow && numOfRows % 100 == 0) { + taosMsleep(200); + } + if (numOfRows > 2000000000) { + simError("script:%s, too many rows return from query", script->fileName); + break; + } + } + + } else { + numOfRows = taos_affected_rows(pSql); + } + + taos_free_result(pSql); + sprintf(script->rows, "%d", numOfRows); + + script->linePos++; + return true; +} + +bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) { + char buf[3000]; + SCmdLine *line = &script->lines[script->linePos]; + + simVisuallizeOption(script, rest, buf); + rest = buf; + + simDebug("script:%s, exec:%s", script->fileName, rest); + strcpy(script->rows, "-1"); + for (int32_t row = 0; row < MAX_QUERY_ROW_NUM; ++row) { + for (int32_t col = 0; col < MAX_QUERY_COL_NUM; ++col) { + strcpy(script->data[row][col], "null"); + } + } + + if (strncmp(rest, "connect", 7) == 0) { + if (!simCreateTaosdConnect(script, rest)) { + return false; + } + script->linePos++; + return true; + } + + if (script->taos == NULL) { + if (!simCreateTaosdConnect(script, "connect root")) { + if (line->errorJump == SQL_JUMP_TRUE) { + script->linePos = line->jump; + return true; + } + return false; + } + } + + if (strncmp(rest, "close", 5) == 0) { + simCloseTaosdConnect(script); + script->linePos++; + return true; + } + + return simExecuteNativeSqlCommand(script, rest, isSlow); +} + +bool simExecuteSqlCmd(SScript *script, char *rest) { + bool isSlow = false; + return simExecuteSqlImpCmd(script, rest, isSlow); +} + +bool simExecuteSqlSlowCmd(SScript *script, char *rest) { + bool isSlow = true; + return simExecuteSqlImpCmd(script, rest, isSlow); +} + +bool simExecuteRestfulCmd(SScript *script, char *rest) { + TdFilePtr pFile = NULL; + char filename[256]; + sprintf(filename, "%s/tmp.sql", simScriptDir); + // fp = fopen(filename, "w"); + pFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + if (pFile == NULL) { + fprintf(stderr, "ERROR: failed to open file: %s\n", filename); + return false; + } + + char db[64] = {0}; + char tb[64] = {0}; + char gzip[32] = {0}; + int32_t ts; + int32_t times; + sscanf(rest, "%s %s %d %d %s", db, tb, &ts, ×, gzip); + + taosFprintfFile(pFile, "insert into %s.%s values ", db, tb); + for (int32_t i = 0; i < times; ++i) { + taosFprintfFile(pFile, "(%d000, %d)", ts + i, ts); + } + taosFprintfFile(pFile, " \n"); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + + char cmd[1024] = {0}; + if (strcmp(gzip, "gzip") == 0) { + sprintf(cmd, + "curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' --header " + "--compressed --data-ascii @%s 127.0.0.1:7111/rest/sql", + filename); + } else { + sprintf(cmd, + "curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' --header " + "'Transfer-Encoding: chunked' --data-ascii @%s 127.0.0.1:7111/rest/sql", + filename); + } + + return simExecuteSystemCmd(script, cmd); +} + +bool simExecuteSqlErrorCmd(SScript *script, char *rest) { + char buf[3000]; + SCmdLine *line = &script->lines[script->linePos]; + + simVisuallizeOption(script, rest, buf); + rest = buf; + + simDebug("script:%s, exec:%s", script->fileName, rest); + strcpy(script->rows, "-1"); + for (int32_t row = 0; row < MAX_QUERY_ROW_NUM; ++row) { + for (int32_t col = 0; col < MAX_QUERY_COL_NUM; ++col) { + strcpy(script->data[row][col], "null"); + } + } + + if (strncmp(rest, "connect", 7) == 0) { + if (!simCreateTaosdConnect(script, rest)) { + return false; + } + script->linePos++; + return true; + } + + if (script->taos == NULL) { + if (!simCreateTaosdConnect(script, "connect root")) { + if (line->errorJump == SQL_JUMP_TRUE) { + script->linePos = line->jump; + return true; + } + return false; + } + } + + if (strncmp(rest, "close", 5) == 0) { + simCloseTaosdConnect(script); + script->linePos++; + return true; + } + + TAOS_RES *pSql = pSql = taos_query(script->taos, rest); + int32_t ret = taos_errno(pSql); + taos_free_result(pSql); + + if (ret != TSDB_CODE_SUCCESS) { + simDebug("script:%s, taos:%p, %s execute, expect failed, so success, ret:%d:%s", script->fileName, script->taos, + rest, ret & 0XFFFF, tstrerror(ret)); + script->linePos++; + return true; + } + + sprintf(script->error, "lineNum:%d. sql:%s expect failed, but success, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, + tstrerror(ret)); + + return false; +} + +bool simExecuteLineInsertCmd(SScript *script, char *rest) { + char buf[TSDB_MAX_BINARY_LEN]; + + simVisuallizeOption(script, rest, buf); + rest = buf; + + SCmdLine *line = &script->lines[script->linePos]; + + simInfo("script:%s, %s", script->fileName, rest); + simLogSql(buf, true); + char *lines[] = {rest}; +#if 0 + int32_t ret = taos_insert_lines(script->taos, lines, 1); +#else + int32_t ret = 0; +#endif + if (ret == TSDB_CODE_SUCCESS) { + simDebug("script:%s, taos:%p, %s executed. success.", script->fileName, script->taos, rest); + script->linePos++; + return true; + } else { + sprintf(script->error, "lineNum: %d. line: %s failed, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, + tstrerror(ret)); + return false; + } +} + +bool simExecuteLineInsertErrorCmd(SScript *script, char *rest) { + char buf[TSDB_MAX_BINARY_LEN]; + + simVisuallizeOption(script, rest, buf); + rest = buf; + + SCmdLine *line = &script->lines[script->linePos]; + + simInfo("script:%s, %s", script->fileName, rest); + simLogSql(buf, true); + char *lines[] = {rest}; +#if 0 + int32_t ret = taos_insert_lines(script->taos, lines, 1); +#else + int32_t ret = 0; +#endif + if (ret == TSDB_CODE_SUCCESS) { + sprintf(script->error, "script:%s, taos:%p, %s executed. expect failed, but success.", script->fileName, + script->taos, rest); + script->linePos++; + return false; + } else { + simDebug("lineNum: %d. line: %s failed, ret:%d:%s. Expect failed, so success", line->lineNum, rest, ret & 0XFFFF, + tstrerror(ret)); + return true; + } +} diff --git a/utils/tsim/src/simMain.c b/utils/tsim/src/simMain.c new file mode 100644 index 0000000000..713e46df58 --- /dev/null +++ b/utils/tsim/src/simMain.c @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "simInt.h" + +bool simExecSuccess = false; +bool abortExecution = false; +bool useMultiProcess = false; +bool useValgrind = false; + +void simHandleSignal(int32_t signo, void *sigInfo, void *context) { + simSystemCleanUp(); + abortExecution = true; +} + +int32_t main(int32_t argc, char *argv[]) { + char scriptFile[MAX_FILE_NAME_LEN] = "sim_main_test.sim"; + + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0 && i < argc - 1) { + tstrncpy(configDir, argv[++i], 128); + } else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) { + strcpy(scriptFile, argv[++i]); + } else if (strcmp(argv[i], "-m") == 0) { + useMultiProcess = true; + } else if (strcmp(argv[i], "-v") == 0) { + useValgrind = true; + } else { + printf("usage: %s [options] \n", argv[0]); + printf(" [-c config]: config directory, default is: %s\n", configDir); + printf(" [-f script]: script filename\n"); + return 0; + } + } + + if (!simSystemInit()) { + simError("failed to initialize the system"); + simSystemCleanUp(); + return -1; + } + + simInfo("simulator is running ..."); + taosSetSignal(SIGINT, simHandleSignal); + + SScript *script = simParseScript(scriptFile); + if (script == NULL) { + simError("parse script file:%s failed", scriptFile); + return -1; + } + + if (abortExecution) { + simError("execute abort"); + return -1; + } + + simScriptList[++simScriptPos] = script; + simExecuteScript(script); + + int32_t ret = simExecSuccess ? 0 : -1; + simInfo("execute result %d", ret); + + return ret; +} diff --git a/utils/tsim/src/simParse.c b/utils/tsim/src/simParse.c new file mode 100644 index 0000000000..b9f7610be8 --- /dev/null +++ b/utils/tsim/src/simParse.c @@ -0,0 +1,1097 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/* Thi file is to parse the simScriptPos, function file and default file, + * + * syntax + * + * expression is format like: + * $var [=|<|>|==] var + * or: + * $var = var [+|-|*|/|.] var + * + * if expression + * command + * + * if expression then + * commands + * elif expression + * commands + * elif expression + * commands + * else + * commands + * endi + * + * while expression + * commands + * continue + * break + * endw + * + * switch expression + * case 1 + * commands + * break + * case 2 + * commands + * break + * default + * commands + * ends + * + * label: + * goto label + * + */ + +#define _DEFAULT_SOURCE +#include "simParse.h" + +static SCommand *cmdHashList[MAX_NUM_CMD]; +static SCmdLine cmdLine[MAX_CMD_LINES]; +static char parseErr[MAX_ERROR_LEN]; +static char optionBuffer[MAX_OPTION_BUFFER]; +static int32_t numOfLines, optionOffset; +static SLabel label, dest; +static SBlock block; + +int32_t simHashCmd(char *token, int32_t tokenLen) { + int32_t i; + int32_t hash = 0; + + for (i = 0; i < tokenLen; ++i) hash += token[i]; + + hash = hash % MAX_NUM_CMD; + + return hash; +} + +SCommand *simCheckCmd(char *token, int32_t tokenLen) { + int32_t hash; + SCommand *node; + + hash = simHashCmd(token, tokenLen); + + node = cmdHashList[hash]; + + while (node) { + if (node->nlen == tokenLen && strncmp(node->name, token, tokenLen) == 0) { + return node; + } else { + node = node->next; + } + } + + return NULL; +} + +void simAddCmdIntoHash(SCommand *pCmd) { + int32_t hash; + SCommand *node; + + hash = simHashCmd(pCmd->name, (int32_t)strlen(pCmd->name)); + node = cmdHashList[hash]; + pCmd->next = node; + cmdHashList[hash] = pCmd; +} + +void simResetParser() { + optionOffset = 4; + numOfLines = 0; + memset(cmdLine, 0, sizeof(cmdLine)); + memset(optionBuffer, 0, sizeof(optionBuffer)); + memset(&label, 0, sizeof(label)); + memset(&block, 0, sizeof(block)); + memset(&dest, 0, sizeof(dest)); +} + +SScript *simBuildScriptObj(char *fileName) { + int32_t i, destPos; + + /* process labels */ + + cmdLine[numOfLines].cmdno = SIM_CMD_RETURN; + numOfLines++; + + for (i = 0; i < numOfLines; ++i) { + cmdLine[i].errorJump = SQL_JUMP_FALSE; + } + + for (--dest.top; dest.top >= 0; --dest.top) { + for (i = 0; i < label.top; ++i) { + if (strcmp(label.label[i], dest.label[(uint8_t)dest.top]) == 0) break; + } + + if (i == label.top) { + sprintf(parseErr, "label:%s not defined", dest.label[(uint8_t)dest.top]); + return NULL; + } + + destPos = dest.pos[(uint8_t)dest.top]; + cmdLine[destPos].jump = label.pos[i]; + if (cmdLine[destPos].cmdno == SIM_CMD_SQL) { + cmdLine[destPos].errorJump = SQL_JUMP_TRUE; + } + } + + if (block.top != 0) { + sprintf(parseErr, "mismatched block"); + return NULL; + } + + for (i = 0; i < numOfLines; ++i) { + if (cmdLine[i].jump == 0) cmdLine[i].jump = numOfLines; + } + + SScript *script = taosMemoryMalloc(sizeof(SScript)); + memset(script, 0, sizeof(SScript)); + + script->type = SIM_SCRIPT_TYPE_MAIN; + script->numOfLines = numOfLines; + tstrncpy(script->fileName, fileName, sizeof(script->fileName)); + + script->optionBuffer = taosMemoryMalloc(optionOffset); + memcpy(script->optionBuffer, optionBuffer, optionOffset); + + script->lines = taosMemoryMalloc(sizeof(SCmdLine) * numOfLines); + memcpy(script->lines, cmdLine, sizeof(SCmdLine) * numOfLines); + + return script; +} + +SScript *simParseScript(char *fileName) { + TdFilePtr pFile; + int32_t tokenLen, lineNum = 0; + char buffer[10*1024], name[128], *token, *rest; + SCommand *pCmd; + SScript *script; + + if ((fileName[0] == '.') || (fileName[0] == '/')) { + strcpy(name, fileName); + } else { + sprintf(name, "%s" TD_DIRSEP "%s", simScriptDir, fileName); + taosRealPath(name, NULL, sizeof(name)); + } + + // if ((fd = fopen(name, "r")) == NULL) { + if ((pFile = taosOpenFile(name, TD_FILE_READ | TD_FILE_STREAM)) == NULL) { + simError("failed to open file:%s", name); + return NULL; + } + + simResetParser(); + + while (!taosEOFFile(pFile)) { + if (taosGetsFile(pFile, sizeof(buffer) - 1, buffer) == -1) continue; + + lineNum++; + int32_t cmdlen = (int32_t)strlen(buffer); + if (buffer[cmdlen - 1] == '\r' || buffer[cmdlen - 1] == '\n') { + buffer[cmdlen - 1] = 0; + } + rest = buffer; + + for (int32_t i = 0; i < cmdlen; ++i) { + if (buffer[i] == '\r' || buffer[i] == '\n') { + buffer[i] = '\0'; + } + } + + again: + rest = paGetToken(rest, &token, &tokenLen); + if (tokenLen == 0) continue; + + if (token[tokenLen - 1] == ':') { + strncpy(label.label[(uint8_t)label.top], token, tokenLen - 1); + label.pos[(uint8_t)label.top] = numOfLines; + label.top++; + goto again; + } + + if (token[0] == '$') { + if (!simParseExpression(token, lineNum)) { + simError("script:%s line:%d %s", fileName, lineNum, parseErr); + return NULL; + } + continue; + } + + if ((pCmd = simCheckCmd(token, tokenLen)) == NULL) { + token[tokenLen] = 0; + simError("script:%s line:%d invalid cmd:%s", fileName, lineNum, token); + return NULL; + } + + if (!pCmd->parseCmd(rest, pCmd, lineNum)) { + simError("script:%s line:%d %s", fileName, lineNum, parseErr); + return NULL; + } + } + taosCloseFile(&pFile); + + script = simBuildScriptObj(fileName); + if (script == NULL) simError("script:%s %s", fileName, parseErr); + + return script; +} + +int32_t simCheckExpression(char *exp) { + char *op1, *op2, *op, *rest; + int32_t op1Len, op2Len, opLen; + + rest = paGetToken(exp, &op1, &op1Len); + if (op1Len == 0) { + sprintf(parseErr, "expression is required"); + return -1; + } + + rest = paGetToken(rest, &op, &opLen); + if (opLen == 0) { + sprintf(parseErr, "operator is missed"); + return -1; + } + + rest = paGetToken(rest, &op2, &op2Len); + if (op2Len == 0) { + sprintf(parseErr, "operand is missed"); + return -1; + } + + if (opLen == 1) { + if (op[0] != '=' && op[0] != '<' && op[0] != '>') { + sprintf(parseErr, "invalid operator:%s", op); + return -1; + } + + if (op[0] == '=' && op1[0] != '$') { + sprintf(parseErr, "left side of assignment must be variable"); + return -1; + } + } else if (opLen == 2) { + if (op[1] != '=' || (op[0] != '=' && op[0] != '<' && op[0] != '>' && op[0] != '!')) { + sprintf(parseErr, "left side of assignment must be variable"); + return -1; + } + } else { + sprintf(parseErr, "invalid operator:%s", op); + return -1; + } + + rest = paGetToken(rest, &op, &opLen); + + if (opLen == 0) return (int32_t)(rest - exp); + + /* if it is key word "then" */ + if (strncmp(op, "then", 4) == 0) return (int32_t)(op - exp); + + rest = paGetToken(rest, &op2, &op2Len); + if (op2Len == 0) { + sprintf(parseErr, "operand is missed"); + return -1; + } + + if (opLen > 1) { + sprintf(parseErr, "invalid operator:%s", op); + return -1; + } + + if (op[0] == '+' || op[0] == '-' || op[0] == '*' || op[0] == '/' || op[0] == '.') { + return (int32_t)(rest - exp); + } + + return -1; +} + +bool simParseExpression(char *token, int32_t lineNum) { + int32_t expLen; + + expLen = simCheckExpression(token); + if (expLen <= 0) return -1; + + cmdLine[numOfLines].cmdno = SIM_CMD_EXP; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + memcpy(optionBuffer + optionOffset, token, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +bool simParseIfCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char *ret; + int32_t expLen; + + expLen = simCheckExpression(rest); + + if (expLen <= 0) return -1; + + ret = rest + expLen; + + if (strncmp(ret, "then", 4) == 0) { + block.type[(uint8_t)block.top] = BLOCK_IF; + block.pos[(uint8_t)block.top] = &cmdLine[numOfLines].jump; + block.top++; + } else { + cmdLine[numOfLines].jump = numOfLines + 2; + } + + cmdLine[numOfLines].optionOffset = optionOffset; + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + cmdLine[numOfLines].cmdno = SIM_CMD_TEST; + cmdLine[numOfLines].lineNum = lineNum; + + numOfLines++; + return true; +} + +bool simParseElifCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; + + expLen = simCheckExpression(rest); + + if (expLen <= 0) return -1; + + if (block.top < 1) { + sprintf(parseErr, "no matching if"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_IF) { + sprintf(parseErr, "no matched if block"); + return false; + } + + cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; + block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump); + block.numJump[block.top - 1]++; + + numOfLines++; + + *(block.pos[block.top - 1]) = numOfLines; + block.pos[block.top - 1] = &cmdLine[numOfLines].jump; + + cmdLine[numOfLines].optionOffset = optionOffset; + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + cmdLine[numOfLines].cmdno = SIM_CMD_TEST; + cmdLine[numOfLines].lineNum = lineNum; + + numOfLines++; + return true; +} + +bool simParseElseCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + if (block.top < 1) { + sprintf(parseErr, "no matching if"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_IF) { + sprintf(parseErr, "no matched if block"); + return false; + } + + cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; + block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump); + block.numJump[block.top - 1]++; + + numOfLines++; + + *(block.pos[block.top - 1]) = numOfLines; + block.pos[block.top - 1] = NULL; + + return true; +} + +bool simParseEndiCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t i; + + if (block.top < 1) { + sprintf(parseErr, "no matching if"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_IF) { + sprintf(parseErr, "no matched if block"); + return false; + } + + if (block.pos[block.top - 1]) *(block.pos[block.top - 1]) = numOfLines; + + for (i = 0; i < block.numJump[block.top - 1]; ++i) { + *(block.jump[block.top - 1][i]) = numOfLines; + } + + block.numJump[block.top - 1] = 0; + block.top--; + + return true; +} + +bool simParseWhileCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; + + expLen = simCheckExpression(rest); + + if (expLen <= 0) return false; + + block.type[(uint8_t)block.top] = BLOCK_WHILE; + block.pos[(uint8_t)block.top] = &(cmdLine[numOfLines].jump); + block.back[(uint8_t)block.top] = numOfLines; + block.top++; + + cmdLine[numOfLines].optionOffset = optionOffset; + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + cmdLine[numOfLines].cmdno = SIM_CMD_TEST; + cmdLine[numOfLines].lineNum = lineNum; + + numOfLines++; + return true; +} + +bool simParseEndwCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t i; + + if (block.top < 1) { + sprintf(parseErr, "no matching while"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_WHILE) { + sprintf(parseErr, "no matched while block"); + return false; + } + + cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; + cmdLine[numOfLines].jump = block.back[block.top - 1]; + cmdLine[numOfLines].lineNum = lineNum; + numOfLines++; + + *(block.pos[block.top - 1]) = numOfLines; + + for (i = 0; i < block.numJump[block.top - 1]; ++i) { + *(block.jump[block.top - 1][i]) = numOfLines; + } + + block.top--; + + return true; +} + +bool simParseSwitchCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char *token; + int32_t tokenLen; + + rest = paGetToken(rest, &token, &tokenLen); + if (tokenLen == 0) { + sprintf(parseErr, "switch should be followed by variable"); + return false; + } + + if (token[0] != '$') { + sprintf(parseErr, "switch must be followed by variable"); + return false; + } + + memcpy(block.sexp[(uint8_t)block.top], token, tokenLen); + block.sexpLen[(uint8_t)block.top] = tokenLen; + block.type[(uint8_t)block.top] = BLOCK_SWITCH; + block.top++; + + return true; +} + +bool simParseCaseCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char *token; + int32_t tokenLen; + + rest = paGetToken(rest, &token, &tokenLen); + if (tokenLen == 0) { + sprintf(parseErr, "case should be followed by value"); + return false; + } + + if (block.top < 1) { + sprintf(parseErr, "no matching switch"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_SWITCH) { + sprintf(parseErr, "case not matched"); + return false; + } + + if (block.pos[block.top - 1] != NULL) { + *(block.pos[block.top - 1]) = numOfLines; + } + + block.pos[block.top - 1] = &(cmdLine[numOfLines].jump); + + cmdLine[numOfLines].cmdno = SIM_CMD_TEST; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + memcpy(optionBuffer + optionOffset, block.sexp[block.top - 1], block.sexpLen[block.top - 1]); + optionOffset += block.sexpLen[block.top - 1]; + *(optionBuffer + optionOffset++) = ' '; + *(optionBuffer + optionOffset++) = '='; + *(optionBuffer + optionOffset++) = '='; + *(optionBuffer + optionOffset++) = ' '; + memcpy(optionBuffer + optionOffset, token, tokenLen); + optionOffset += tokenLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +bool simParseBreakCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + if (block.top < 1) { + sprintf(parseErr, "no blcok exists"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_SWITCH && block.type[block.top - 1] != BLOCK_WHILE) { + sprintf(parseErr, "not in switch or while block"); + return false; + } + + block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump); + block.numJump[block.top - 1]++; + + cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; + cmdLine[numOfLines].lineNum = lineNum; + + numOfLines++; + return true; +} + +bool simParseDefaultCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + if (block.top < 1) { + sprintf(parseErr, "no matching switch"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_SWITCH) { + sprintf(parseErr, "default should be matched with switch"); + return false; + } + + if (block.pos[block.top - 1] != NULL) { + *(block.pos[block.top - 1]) = numOfLines; + } + + return true; +} + +bool simParseEndsCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t i; + + if (block.top < 1) { + sprintf(parseErr, "no matching switch"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_SWITCH) { + sprintf(parseErr, "ends should be matched with switch"); + return false; + } + + for (i = 0; i < block.numJump[block.top - 1]; ++i) { + *(block.jump[block.top - 1][i]) = numOfLines; + } + + block.numJump[block.top - 1] = 0; + block.top--; + + return true; +} + +bool simParseContinueCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + if (block.top < 1) { + sprintf(parseErr, "no matching while"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_WHILE) { + sprintf(parseErr, "continue should be matched with while cmd"); + return false; + } + + cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].jump = block.back[block.top - 1]; + + numOfLines++; + return true; +} + +bool simParsePrintCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; + + rest++; + cmdLine[numOfLines].cmdno = SIM_CMD_PRINT; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + expLen = (int32_t)strlen(rest); + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +void simCheckSqlOption(char *rest) { + int32_t valueLen; + char *value, *xpos; + + xpos = strstr(rest, " -x"); // need a blank + if (xpos) { + paGetToken(xpos + 3, &value, &valueLen); + if (valueLen != 0) { + memcpy(dest.label[(uint8_t)dest.top], value, valueLen); + dest.label[(uint8_t)dest.top][valueLen] = 0; + dest.pos[(uint8_t)dest.top] = numOfLines; + dest.top++; + + *xpos = 0; + } + } +} + +bool simParseSqlCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; + + rest++; + simCheckSqlOption(rest); + cmdLine[numOfLines].cmdno = SIM_CMD_SQL; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + expLen = (int32_t)strlen(rest); + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +bool simParseSqlErrorCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; + + rest++; + cmdLine[numOfLines].cmdno = SIM_CMD_SQL_ERROR; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + expLen = (int32_t)strlen(rest); + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +bool simParseSqlSlowCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + simParseSqlCmd(rest, pCmd, lineNum); + cmdLine[numOfLines - 1].cmdno = SIM_CMD_SQL_SLOW; + return true; +} + +bool simParseRestfulCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + simParseSqlCmd(rest, pCmd, lineNum); + cmdLine[numOfLines - 1].cmdno = SIM_CMD_RESTFUL; + return true; +} + +bool simParseSystemCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; + + rest++; + cmdLine[numOfLines].cmdno = SIM_CMD_SYSTEM; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + expLen = (int32_t)strlen(rest); + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +bool simParseSystemContentCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + simParseSystemCmd(rest, pCmd, lineNum); + cmdLine[numOfLines - 1].cmdno = SIM_CMD_SYSTEM_CONTENT; + return true; +} + +bool simParseSleepCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char *token; + int32_t tokenLen; + + cmdLine[numOfLines].cmdno = SIM_CMD_SLEEP; + cmdLine[numOfLines].lineNum = lineNum; + + paGetToken(rest, &token, &tokenLen); + if (tokenLen > 0) { + cmdLine[numOfLines].optionOffset = optionOffset; + memcpy(optionBuffer + optionOffset, token, tokenLen); + optionOffset += tokenLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + } + + numOfLines++; + return true; +} + +bool simParseReturnCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char *token; + int32_t tokenLen; + + cmdLine[numOfLines].cmdno = SIM_CMD_RETURN; + cmdLine[numOfLines].lineNum = lineNum; + + paGetToken(rest, &token, &tokenLen); + if (tokenLen > 0) { + cmdLine[numOfLines].optionOffset = optionOffset; + memcpy(optionBuffer + optionOffset, token, tokenLen); + optionOffset += tokenLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + } + + numOfLines++; + return true; +} + +bool simParseGotoCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char *token; + int32_t tokenLen; + + rest = paGetToken(rest, &token, &tokenLen); + + if (tokenLen == 0) { + sprintf(parseErr, "label should be followed by goto cmd"); + return false; + } + + memcpy(dest.label[(uint8_t)dest.top], token, tokenLen); + dest.label[(uint8_t)dest.top][tokenLen] = 0; + dest.pos[(uint8_t)dest.top] = numOfLines; + dest.top++; + + cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; + cmdLine[numOfLines].lineNum = lineNum; + + numOfLines++; + return true; +} + +bool simParseRunCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char *token; + int32_t tokenLen; + + rest = paGetToken(rest, &token, &tokenLen); + + if (tokenLen == 0) { + sprintf(parseErr, "file name should be followed by run cmd"); + return false; + } + + cmdLine[numOfLines].cmdno = SIM_CMD_RUN; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + memcpy(optionBuffer + optionOffset, token, tokenLen); + optionOffset += tokenLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +bool simParseRunBackCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + simParseRunCmd(rest, pCmd, lineNum); + cmdLine[numOfLines - 1].cmdno = SIM_CMD_RUN_BACK; + return true; +} + +bool simParseLineInsertCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; + + rest++; + cmdLine[numOfLines].cmdno = SIM_CMD_LINE_INSERT; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + expLen = (int32_t)strlen(rest); + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +bool simParseLineInsertErrorCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; + + rest++; + cmdLine[numOfLines].cmdno = SIM_CMD_LINE_INSERT; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + expLen = (int32_t)strlen(rest); + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +void simInitsimCmdList() { + int32_t cmdno; + memset(simCmdList, 0, SIM_CMD_END * sizeof(SCommand)); + + /* internal command */ + cmdno = SIM_CMD_EXP; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "exp"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = NULL; + simCmdList[cmdno].executeCmd = simExecuteExpCmd; + + cmdno = SIM_CMD_IF; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "if"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseIfCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_ELIF; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "elif"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseElifCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_ELSE; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "else"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseElseCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_ENDI; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "endi"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseEndiCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_WHILE; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "while"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseWhileCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_ENDW; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "endw"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseEndwCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_SWITCH; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "switch"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseSwitchCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_CASE; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "case"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseCaseCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_DEFAULT; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "default"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseDefaultCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_BREAK; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "break"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseBreakCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_CONTINUE; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "continue"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseContinueCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_ENDS; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "ends"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseEndsCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_SLEEP; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "sleep"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseSleepCmd; + simCmdList[cmdno].executeCmd = simExecuteSleepCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_GOTO; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "goto"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseGotoCmd; + simCmdList[cmdno].executeCmd = simExecuteGotoCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_RUN; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "run"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseRunCmd; + simCmdList[cmdno].executeCmd = simExecuteRunCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_RUN_BACK; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "run_back"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseRunBackCmd; + simCmdList[cmdno].executeCmd = simExecuteRunBackCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_SYSTEM; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "system"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseSystemCmd; + simCmdList[cmdno].executeCmd = simExecuteSystemCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_SYSTEM_CONTENT; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "system_content"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseSystemContentCmd; + simCmdList[cmdno].executeCmd = simExecuteSystemContentCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_PRINT; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "print"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParsePrintCmd; + simCmdList[cmdno].executeCmd = simExecutePrintCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_SQL; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "sql"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseSqlCmd; + simCmdList[cmdno].executeCmd = simExecuteSqlCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_SQL_ERROR; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "sql_error"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseSqlErrorCmd; + simCmdList[cmdno].executeCmd = simExecuteSqlErrorCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_SQL_SLOW; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "sql_slow"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseSqlSlowCmd; + simCmdList[cmdno].executeCmd = simExecuteSqlSlowCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_RESTFUL; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "restful"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseRestfulCmd; + simCmdList[cmdno].executeCmd = simExecuteRestfulCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + /* test is only an internal command */ + cmdno = SIM_CMD_TEST; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "test"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = NULL; + simCmdList[cmdno].executeCmd = simExecuteTestCmd; + + cmdno = SIM_CMD_RETURN; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "return"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseReturnCmd; + simCmdList[cmdno].executeCmd = simExecuteReturnCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_LINE_INSERT; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "line_insert"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseLineInsertCmd; + simCmdList[cmdno].executeCmd = simExecuteLineInsertCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_LINE_INSERT_ERROR; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "line_insert_error"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseLineInsertErrorCmd; + simCmdList[cmdno].executeCmd = simExecuteLineInsertErrorCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); +} diff --git a/utils/tsim/src/simSystem.c b/utils/tsim/src/simSystem.c new file mode 100644 index 0000000000..f2fefb903d --- /dev/null +++ b/utils/tsim/src/simSystem.c @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "simInt.h" +#include "tconfig.h" + +SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; +SCommand simCmdList[SIM_CMD_END]; +int32_t simScriptPos = -1; +int32_t simScriptSucced = 0; +int32_t simDebugFlag = 143; +void simCloseTaosdConnect(SScript *script); +char simScriptDir[PATH_MAX] = {0}; + +extern bool simExecSuccess; + +int32_t simInitCfg() { + taosCreateLog("simlog", 1, configDir, NULL, NULL, NULL, NULL, 1); + taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1); + + SConfig *pCfg = taosGetCfg(); + simDebugFlag = cfgGetItem(pCfg, "simDebugFlag")->i32; + tstrncpy(simScriptDir, cfgGetItem(pCfg, "scriptDir")->str, PATH_MAX); + return 0; +} + +bool simSystemInit() { + simInitCfg(); + simInitsimCmdList(); + memset(simScriptList, 0, sizeof(SScript *) * MAX_MAIN_SCRIPT_NUM); + return true; +} + +void simSystemCleanUp() {} + +void simFreeScript(SScript *script) { + if (script->type == SIM_SCRIPT_TYPE_MAIN) { + simInfo("script:%s, background script num:%d, stop them", script->fileName, script->bgScriptLen); + + for (int32_t i = 0; i < script->bgScriptLen; ++i) { + SScript *bgScript = script->bgScripts[i]; + simDebug("script:%s, is background script, set stop flag", bgScript->fileName); + bgScript->killed = true; + if (taosCheckPthreadValid(bgScript->bgPid)) { + taosThreadJoin(bgScript->bgPid, NULL); + taosThreadClear(&bgScript->bgPid); + } + + simDebug("script:%s, background thread joined", bgScript->fileName); + taos_close(bgScript->taos); + taosMemoryFreeClear(bgScript->lines); + taosMemoryFreeClear(bgScript->optionBuffer); + taosMemoryFreeClear(bgScript); + } + + simDebug("script:%s, is cleaned", script->fileName); + taos_close(script->taos); + taosMemoryFreeClear(script->lines); + taosMemoryFreeClear(script->optionBuffer); + taosMemoryFreeClear(script); + } +} + +SScript *simProcessCallOver(SScript *script) { + if (script->type == SIM_SCRIPT_TYPE_MAIN) { + simDebug("script:%s, is main script, set stop flag", script->fileName); + if (script->killed) { + simExecSuccess = false; + simInfo("script:" FAILED_PREFIX "%s" FAILED_POSTFIX ", " FAILED_PREFIX "failed" FAILED_POSTFIX ", error:%s", + script->fileName, script->error); + } else { + simExecSuccess = true; + simInfo("script:" SUCCESS_PREFIX "%s" SUCCESS_POSTFIX ", " SUCCESS_PREFIX "success" SUCCESS_POSTFIX, + script->fileName); + } + + simCloseTaosdConnect(script); + simScriptSucced++; + simScriptPos--; + simFreeScript(script); + + if (simScriptPos == -1 && simExecSuccess) { + simInfo("----------------------------------------------------------------------"); + simInfo("Simulation Test Done, " SUCCESS_PREFIX "%d" SUCCESS_POSTFIX " Passed:\n", simScriptSucced); + return NULL; + } + + if (simScriptPos == -1) return NULL; + if (!simExecSuccess) return NULL; + + return simScriptList[simScriptPos]; + } else { + simDebug("script:%s, is stopped", script->fileName); + simFreeScript(script); + return NULL; + } +} + +void *simExecuteScript(void *inputScript) { + SScript *script = (SScript *)inputScript; + + while (1) { + if (script->type == SIM_SCRIPT_TYPE_MAIN) { + script = simScriptList[simScriptPos]; + } + + if (abortExecution) { + script->killed = true; + } + + if (script->killed || script->linePos >= script->numOfLines) { + script = simProcessCallOver(script); + if (script == NULL) { + simDebug("sim test abort now!"); + break; + } + } else { + SCmdLine *line = &script->lines[script->linePos]; + char *option = script->optionBuffer + line->optionOffset; + simDebug("script:%s, line:%d with option \"%s\"", script->fileName, line->lineNum, option); + + SCommand *cmd = &simCmdList[line->cmdno]; + int32_t ret = (*(cmd->executeCmd))(script, option); + if (!ret) { + script->killed = true; + } + } + } + + simInfo("thread is stopped"); + return NULL; +} From de09dadc8299bdd09ff40970326663835b9417fe Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 3 Sep 2022 08:31:41 +0800 Subject: [PATCH 12/16] chore: tests reorg (#16638) * chore: move few tests items out * fix: add taos as dep to tmq_demo * chore: remove tests/test and tests/tsim --- CMakeLists.txt | 1 - tests/test/c/CMakeLists.txt | 59 -- tests/test/c/createTable.c | 471 ----------- tests/test/c/sdbDump.c | 475 ----------- tests/test/c/sml_test.c | 1191 --------------------------- tests/test/c/tmqDemo.c | 705 ---------------- tests/test/c/tmqSim.c | 1520 ----------------------------------- tests/test/c/tmq_taosx_ci.c | 623 -------------- tests/tsim/CMakeLists.txt | 14 - tests/tsim/inc/simInt.h | 186 ----- tests/tsim/inc/simParse.h | 57 -- tests/tsim/src/simExe.c | 1016 ----------------------- tests/tsim/src/simMain.c | 76 -- tests/tsim/src/simParse.c | 1097 ------------------------- tests/tsim/src/simSystem.c | 145 ---- 15 files changed, 7636 deletions(-) delete mode 100644 tests/test/c/CMakeLists.txt delete mode 100644 tests/test/c/createTable.c delete mode 100644 tests/test/c/sdbDump.c delete mode 100644 tests/test/c/sml_test.c delete mode 100644 tests/test/c/tmqDemo.c delete mode 100644 tests/test/c/tmqSim.c delete mode 100644 tests/test/c/tmq_taosx_ci.c delete mode 100644 tests/tsim/CMakeLists.txt delete mode 100644 tests/tsim/inc/simInt.h delete mode 100644 tests/tsim/inc/simParse.h delete mode 100644 tests/tsim/src/simExe.c delete mode 100644 tests/tsim/src/simMain.c delete mode 100644 tests/tsim/src/simParse.c delete mode 100644 tests/tsim/src/simSystem.c diff --git a/CMakeLists.txt b/CMakeLists.txt index e534f925aa..90e841d5e0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,7 +34,6 @@ endif(${BUILD_TEST}) add_subdirectory(source) add_subdirectory(tools) -#add_subdirectory(tests) add_subdirectory(utils) add_subdirectory(examples/c) diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt deleted file mode 100644 index 839c65b633..0000000000 --- a/tests/test/c/CMakeLists.txt +++ /dev/null @@ -1,59 +0,0 @@ -add_executable(tmq_demo tmqDemo.c) -add_dependencies(tmq_demo taos) -add_executable(tmq_sim tmqSim.c) -add_executable(create_table createTable.c) -add_executable(tmq_taosx_ci tmq_taosx_ci.c) -add_executable(sml_test sml_test.c) -target_link_libraries( - create_table - PUBLIC taos_static - PUBLIC util - PUBLIC common - PUBLIC os -) -target_link_libraries( - tmq_demo - PUBLIC taos_static - PUBLIC util - PUBLIC common - PUBLIC os -) -target_link_libraries( - tmq_sim - PUBLIC taos_static - PUBLIC util - PUBLIC common - PUBLIC os -) -target_link_libraries( - tmq_taosx_ci - PUBLIC taos_static - PUBLIC util - PUBLIC common - PUBLIC os -) - -target_link_libraries( - sml_test - PUBLIC taos_static - PUBLIC util - PUBLIC common - PUBLIC os -) - -add_executable(sdbDump sdbDump.c) -target_link_libraries( - sdbDump - PUBLIC dnode - PUBLIC mnode - PUBLIC stream - PUBLIC sdb - PUBLIC os -) -target_include_directories( - sdbDump - PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode" - PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/impl/inc" - PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/sdb/inc" - PRIVATE "${TD_SOURCE_DIR}/source/dnode/mgmt/node_mgmt/inc" -) diff --git a/tests/test/c/createTable.c b/tests/test/c/createTable.c deleted file mode 100644 index 6a0f8e244e..0000000000 --- a/tests/test/c/createTable.c +++ /dev/null @@ -1,471 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taos.h" -#include "taoserror.h" -#include "tlog.h" - -#define GREEN "\033[1;32m" -#define NC "\033[0m" - -char dbName[32] = "db"; -char stbName[64] = "st"; -int32_t numOfThreads = 1; -int64_t numOfTables = 200000; -int64_t startOffset = 0; -int32_t createTable = 1; -int32_t insertData = 0; -int32_t batchNumOfTbl = 100; -int32_t batchNumOfRow = 1; -int32_t totalRowsOfPerTbl = 1; -int32_t numOfVgroups = 2; -int32_t showTablesFlag = 0; -int32_t queryFlag = 0; - -int64_t startTimestamp = 1640966400000; // 2020-01-01 00:00:00.000 - -typedef struct { - int64_t tableBeginIndex; - int64_t tableEndIndex; - int32_t threadIndex; - char dbName[32]; - char stbName[64]; - float createTableSpeed; - float insertDataSpeed; - int64_t startMs; - int64_t maxDelay; - int64_t minDelay; - TdThread thread; -} SThreadInfo; - -// void parseArgument(int32_t argc, char *argv[]); -// void *threadFunc(void *param); -// void createDbAndStb(); - -void createDbAndStb() { - pPrint("start to create db and stable"); - char qstr[64000]; - - TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); - if (con == NULL) { - pError("failed to connect to DB, reason:%s", taos_errstr(NULL)); - exit(1); - } - - sprintf(qstr, "create database if not exists %s vgroups %d", dbName, numOfVgroups); - TAOS_RES *pRes = taos_query(con, qstr); - int32_t code = taos_errno(pRes); - if (code != 0) { - pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(pRes), - taos_errstr(pRes)); - exit(0); - } - taos_free_result(pRes); - - sprintf(qstr, "use %s", dbName); - pRes = taos_query(con, qstr); - code = taos_errno(pRes); - if (code != 0) { - pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); - exit(0); - } - taos_free_result(pRes); - - sprintf(qstr, "create table if not exists %s (ts timestamp, i int) tags (j bigint)", stbName); - pRes = taos_query(con, qstr); - code = taos_errno(pRes); - if (code != 0) { - pError("failed to create stable, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); - exit(0); - } - taos_free_result(pRes); - - taos_close(con); -} - -void printCreateProgress(SThreadInfo *pInfo, int64_t t) { - int64_t endMs = taosGetTimestampMs(); - int64_t totalTables = t - pInfo->tableBeginIndex; - float seconds = (endMs - pInfo->startMs) / 1000.0; - float speed = totalTables / seconds; - pInfo->createTableSpeed = speed; - pPrint("thread:%d, %" PRId64 " tables created, time:%.2f sec, speed:%.1f tables/second, ", pInfo->threadIndex, - totalTables, seconds, speed); -} - -void printInsertProgress(SThreadInfo *pInfo, int64_t insertTotalRows) { - int64_t endMs = taosGetTimestampMs(); - //int64_t totalTables = t - pInfo->tableBeginIndex; - float seconds = (endMs - pInfo->startMs) / 1000.0; - float speed = insertTotalRows / seconds; - pInfo->insertDataSpeed = speed; - pPrint("thread:%d, %" PRId64 " rows inserted, time:%.2f sec, speed:%.1f rows/second, ", pInfo->threadIndex, - insertTotalRows, seconds, speed); -} - -static int64_t getResult(TAOS_RES *tres) { - TAOS_ROW row = taos_fetch_row(tres); - if (row == NULL) { - return 0; - } - - int num_fields = taos_num_fields(tres); - TAOS_FIELD *fields = taos_fetch_fields(tres); - int precision = taos_result_precision(tres); - - int64_t numOfRows = 0; - do { - numOfRows++; - row = taos_fetch_row(tres); - } while (row != NULL); - - return numOfRows; -} - -void showTables() { - pPrint("start to show tables"); - char qstr[128]; - - TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); - if (con == NULL) { - pError("failed to connect to DB, reason:%s", taos_errstr(NULL)); - exit(1); - } - - snprintf(qstr, 128, "use %s", dbName); - TAOS_RES *pRes = taos_query(con, qstr); - int code = taos_errno(pRes); - if (code != 0) { - pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); - exit(1); - } - taos_free_result(pRes); - - sprintf(qstr, "show tables"); - pRes = taos_query(con, qstr); - code = taos_errno(pRes); - if (code != 0) { - pError("failed to show tables, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); - exit(0); - } - - int64_t totalTableNum = getResult(pRes); - taos_free_result(pRes); - - pPrint("%s database: %s, total %" PRId64 " tables %s", GREEN, dbName, totalTableNum, NC); - - taos_close(con); -} - -void *threadFunc(void *param) { - SThreadInfo *pInfo = (SThreadInfo *)param; - char *qstr = taosMemoryMalloc(batchNumOfTbl * batchNumOfRow * 128); - int32_t code = 0; - - TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); - if (con == NULL) { - pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(NULL)); - exit(1); - } - - //pPrint("====before thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, - // pInfo->tableEndIndex); - - pInfo->tableBeginIndex += startOffset; - pInfo->tableEndIndex += startOffset; - - pPrint("====thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, pInfo->tableEndIndex); - - sprintf(qstr, "use %s", pInfo->dbName); - TAOS_RES *pRes = taos_query(con, qstr); - taos_free_result(pRes); - - if (createTable) { - int64_t curMs = 0; - int64_t beginMs = taosGetTimestampMs(); - pInfo->startMs = beginMs; - int64_t t = pInfo->tableBeginIndex; - for (; t <= pInfo->tableEndIndex;) { - // int64_t batch = (pInfo->tableEndIndex - t); - // batch = MIN(batch, batchNum); - - int32_t len = sprintf(qstr, "create table"); - for (int32_t i = 0; i < batchNumOfTbl;) { - len += sprintf(qstr + len, " %s_t%" PRId64 " using %s tags(%" PRId64 ")", stbName, t, stbName, t); - t++; - i++; - if (t > pInfo->tableEndIndex) { - break; - } - } - - int64_t startTs = taosGetTimestampUs(); - TAOS_RES *pRes = taos_query(con, qstr); - code = taos_errno(pRes); - if (code != 0) { - pError("failed to create table reason:%s, sql: %s", tstrerror(code), qstr); - } - taos_free_result(pRes); - int64_t endTs = taosGetTimestampUs(); - int64_t delay = endTs - startTs; - // printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay); - if (delay > pInfo->maxDelay) pInfo->maxDelay = delay; - if (delay < pInfo->minDelay) pInfo->minDelay = delay; - - curMs = taosGetTimestampMs(); - if (curMs - beginMs > 10000) { - beginMs = curMs; - // printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t); - printCreateProgress(pInfo, t); - } - } - printCreateProgress(pInfo, t); - } - - if (insertData) { - int64_t insertTotalRows = 0; - int64_t curMs = 0; - int64_t beginMs = taosGetTimestampMs(); - pInfo->startMs = beginMs; - int64_t t = pInfo->tableBeginIndex; - for (; t <= pInfo->tableEndIndex; t++) { - //printf("table name: %"PRId64"\n", t); - int64_t ts = startTimestamp; - for (int32_t i = 0; i < totalRowsOfPerTbl;) { - int32_t len = sprintf(qstr, "insert into "); - len += sprintf(qstr + len, "%s_t%" PRId64 " values ", stbName, t); - for (int32_t j = 0; j < batchNumOfRow; j++) { - len += sprintf(qstr + len, "(%" PRId64 ", 6666) ", ts++); - i++; - insertTotalRows++; - if (i >= totalRowsOfPerTbl) { - break; - } - } - - #if 1 - int64_t startTs = taosGetTimestampUs(); - TAOS_RES *pRes = taos_query(con, qstr); - code = taos_errno(pRes); - if (code != 0) { - pError("failed to insert %s_t%" PRId64 ", reason:%s", stbName, t, tstrerror(code)); - } - taos_free_result(pRes); - int64_t endTs = taosGetTimestampUs(); - int64_t delay = endTs - startTs; - // printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay); - if (delay > pInfo->maxDelay) pInfo->maxDelay = delay; - if (delay < pInfo->minDelay) pInfo->minDelay = delay; - - curMs = taosGetTimestampMs(); - if (curMs - beginMs > 10000) { - beginMs = curMs; - // printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t); - printInsertProgress(pInfo, insertTotalRows); - } - #endif - } - } - printInsertProgress(pInfo, insertTotalRows); - } - - taos_close(con); - taosMemoryFree(qstr); - return 0; -} - -void printHelp() { - char indent[10] = " "; - printf("Used to test the performance while create table\n"); - - printf("%s%s\n", indent, "-c"); - printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); - printf("%s%s\n", indent, "-d"); - printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName); - printf("%s%s\n", indent, "-s"); - printf("%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", stbName); - printf("%s%s\n", indent, "-t"); - printf("%s%s%s%d\n", indent, indent, "numOfThreads, default is ", numOfThreads); - printf("%s%s\n", indent, "-n"); - printf("%s%s%s%" PRId64 "\n", indent, indent, "numOfTables, default is ", numOfTables); - printf("%s%s\n", indent, "-g"); - printf("%s%s%s%" PRId64 "\n", indent, indent, "startOffset, default is ", startOffset); - printf("%s%s\n", indent, "-v"); - printf("%s%s%s%d\n", indent, indent, "numOfVgroups, default is ", numOfVgroups); - printf("%s%s\n", indent, "-a"); - printf("%s%s%s%d\n", indent, indent, "createTable, default is ", createTable); - printf("%s%s\n", indent, "-i"); - printf("%s%s%s%d\n", indent, indent, "insertData, default is ", insertData); - printf("%s%s\n", indent, "-b"); - printf("%s%s%s%d\n", indent, indent, "batchNumOfTbl, default is ", batchNumOfTbl); - printf("%s%s\n", indent, "-w"); - printf("%s%s%s%d\n", indent, indent, "showTablesFlag, default is ", showTablesFlag); - printf("%s%s\n", indent, "-q"); - printf("%s%s%s%d\n", indent, indent, "queryFlag, default is ", queryFlag); - printf("%s%s\n", indent, "-l"); - printf("%s%s%s%d\n", indent, indent, "batchNumOfRow, default is ", batchNumOfRow); - printf("%s%s\n", indent, "-r"); - printf("%s%s%s%d\n", indent, indent, "totalRowsOfPerTbl, default is ", totalRowsOfPerTbl); - - exit(EXIT_SUCCESS); -} - -void parseArgument(int32_t argc, char *argv[]) { - for (int32_t i = 1; i < argc; i++) { - if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { - printHelp(); - exit(0); - } else if (strcmp(argv[i], "-d") == 0) { - strcpy(dbName, argv[++i]); - } else if (strcmp(argv[i], "-c") == 0) { - strcpy(configDir, argv[++i]); - } else if (strcmp(argv[i], "-s") == 0) { - strcpy(stbName, argv[++i]); - } else if (strcmp(argv[i], "-t") == 0) { - numOfThreads = atoi(argv[++i]); - } else if (strcmp(argv[i], "-n") == 0) { - numOfTables = atoll(argv[++i]); - } else if (strcmp(argv[i], "-g") == 0) { - startOffset = atoll(argv[++i]); - } else if (strcmp(argv[i], "-v") == 0) { - numOfVgroups = atoi(argv[++i]); - } else if (strcmp(argv[i], "-a") == 0) { - createTable = atoi(argv[++i]); - } else if (strcmp(argv[i], "-i") == 0) { - insertData = atoi(argv[++i]); - } else if (strcmp(argv[i], "-b") == 0) { - batchNumOfTbl = atoi(argv[++i]); - } else if (strcmp(argv[i], "-l") == 0) { - batchNumOfRow = atoi(argv[++i]); - } else if (strcmp(argv[i], "-r") == 0) { - totalRowsOfPerTbl = atoi(argv[++i]); - } else if (strcmp(argv[i], "-w") == 0) { - showTablesFlag = atoi(argv[++i]); - } else if (strcmp(argv[i], "-q") == 0) { - queryFlag = atoi(argv[++i]); - } else { - pPrint("%s unknow para: %s %s", GREEN, argv[++i], NC); - } - } - - pPrint("%s dbName:%s %s", GREEN, dbName, NC); - pPrint("%s stbName:%s %s", GREEN, stbName, NC); - pPrint("%s configDir:%s %s", GREEN, configDir, NC); - pPrint("%s numOfTables:%" PRId64 " %s", GREEN, numOfTables, NC); - pPrint("%s startOffset:%" PRId64 " %s", GREEN, startOffset, NC); - pPrint("%s numOfThreads:%d %s", GREEN, numOfThreads, NC); - pPrint("%s numOfVgroups:%d %s", GREEN, numOfVgroups, NC); - pPrint("%s createTable:%d %s", GREEN, createTable, NC); - pPrint("%s insertData:%d %s", GREEN, insertData, NC); - pPrint("%s batchNumOfTbl:%d %s", GREEN, batchNumOfTbl, NC); - pPrint("%s batchNumOfRow:%d %s", GREEN, batchNumOfRow, NC); - pPrint("%s totalRowsOfPerTbl:%d %s", GREEN, totalRowsOfPerTbl, NC); - pPrint("%s showTablesFlag:%d %s", GREEN, showTablesFlag, NC); - pPrint("%s queryFlag:%d %s", GREEN, queryFlag, NC); - - pPrint("%s start create table performace test %s", GREEN, NC); -} - -int32_t main(int32_t argc, char *argv[]) { - parseArgument(argc, argv); - - if (showTablesFlag) { - showTables(); - return 0; - } - - if (queryFlag) { - // selectRowsFromTable(); - return 0; - } - - if (createTable) { - createDbAndStb(); - } - - pPrint("%d threads are spawned to create %" PRId64 " tables, offset is %" PRId64 " ", numOfThreads, numOfTables, - startOffset); - - TdThreadAttr thattr; - taosThreadAttrInit(&thattr); - taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - SThreadInfo *pInfo = (SThreadInfo *)taosMemoryCalloc(numOfThreads, sizeof(SThreadInfo)); - - // int64_t numOfTablesPerThread = numOfTables / numOfThreads; - // numOfTables = numOfTablesPerThread * numOfThreads; - - if (numOfThreads < 1) { - numOfThreads = 1; - } - - int64_t a = numOfTables / numOfThreads; - if (a < 1) { - numOfThreads = numOfTables; - a = 1; - } - - int64_t b = 0; - b = numOfTables % numOfThreads; - - int64_t tableFrom = 0; - for (int32_t i = 0; i < numOfThreads; ++i) { - pInfo[i].tableBeginIndex = tableFrom; - pInfo[i].tableEndIndex = (i < b ? tableFrom + a : tableFrom + a - 1); - tableFrom = pInfo[i].tableEndIndex + 1; - pInfo[i].threadIndex = i; - pInfo[i].minDelay = INT64_MAX; - strcpy(pInfo[i].dbName, dbName); - strcpy(pInfo[i].stbName, stbName); - taosThreadCreate(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); - } - - taosMsleep(300); - for (int32_t i = 0; i < numOfThreads; i++) { - taosThreadJoin(pInfo[i].thread, NULL); - taosThreadClear(&pInfo[i].thread); - } - - int64_t maxDelay = 0; - int64_t minDelay = INT64_MAX; - - float createTableSpeed = 0; - for (int32_t i = 0; i < numOfThreads; ++i) { - createTableSpeed += pInfo[i].createTableSpeed; - - if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay; - if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay; - } - - float insertDataSpeed = 0; - for (int32_t i = 0; i < numOfThreads; ++i) { - insertDataSpeed += pInfo[i].insertDataSpeed; - } - - if (createTable) { - pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 - "us %s", - GREEN, numOfTables, createTableSpeed, numOfThreads, maxDelay, minDelay, NC); - } - - if (insertData) { - pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, - numOfThreads, NC); - } - - taosThreadAttrDestroy(&thattr); - taosMemoryFree(pInfo); -} diff --git a/tests/test/c/sdbDump.c b/tests/test/c/sdbDump.c deleted file mode 100644 index b90b781e44..0000000000 --- a/tests/test/c/sdbDump.c +++ /dev/null @@ -1,475 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dmMgmt.h" -#include "mndInt.h" -#include "sdb.h" -#include "tconfig.h" -#include "tjson.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-result" - -#define TMP_DNODE_DIR TD_TMP_DIR_PATH "dumpsdb" -#define TMP_MNODE_DIR TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" -#define TMP_SDB_DATA_DIR TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "data" -#define TMP_SDB_SYNC_DIR TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "sync" -#define TMP_SDB_MNODE_JSON TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "mnode.json" -#define TMP_SDB_DATA_FILE TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "data" TD_DIRSEP "sdb.data" -#define TMP_SDB_RAFT_CFG_FILE TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_config.json" -#define TMP_SDB_RAFT_STORE_FILE TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_store.json" - -void reportStartup(const char *name, const char *desc) {} - -void sendRsp(SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); } - -int32_t sendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { - terrno = TSDB_CODE_INVALID_PTR; - return -1; -} - -char *i642str(int64_t val) { - static char str[24] = {0}; - snprintf(str, sizeof(str), "%" PRId64, val); - return str; -} - -void dumpFunc(SSdb *pSdb, SJson *json) {} - -void dumpDb(SSdb *pSdb, SJson *json) { - void *pIter = NULL; - SJson *items = tjsonCreateObject(); - tjsonAddItemToObject(json, "dbs", items); - - while (1) { - SDbObj *pObj = NULL; - pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pObj); - if (pIter == NULL) break; - - SJson *item = tjsonCreateObject(); - tjsonAddItemToObject(items, "db", item); - - tjsonAddStringToObject(item, "name", pObj->name); - tjsonAddStringToObject(item, "acct", pObj->acct); - tjsonAddStringToObject(item, "createUser", pObj->createUser); - tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); - tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); - tjsonAddStringToObject(item, "uid", i642str(pObj->uid)); - tjsonAddIntegerToObject(item, "cfgVersion", pObj->cfgVersion); - tjsonAddIntegerToObject(item, "vgVersion", pObj->vgVersion); - tjsonAddIntegerToObject(item, "numOfVgroups", pObj->cfg.numOfVgroups); - tjsonAddIntegerToObject(item, "numOfStables", pObj->cfg.numOfStables); - tjsonAddIntegerToObject(item, "buffer", pObj->cfg.buffer); - tjsonAddIntegerToObject(item, "pageSize", pObj->cfg.pageSize); - tjsonAddIntegerToObject(item, "pages", pObj->cfg.pages); - tjsonAddIntegerToObject(item, "cacheLastSize", pObj->cfg.cacheLastSize); - tjsonAddIntegerToObject(item, "daysPerFile", pObj->cfg.daysPerFile); - tjsonAddIntegerToObject(item, "daysToKeep0", pObj->cfg.daysToKeep0); - tjsonAddIntegerToObject(item, "daysToKeep1", pObj->cfg.daysToKeep1); - tjsonAddIntegerToObject(item, "daysToKeep2", pObj->cfg.daysToKeep2); - tjsonAddIntegerToObject(item, "minRows", pObj->cfg.minRows); - tjsonAddIntegerToObject(item, "maxRows", pObj->cfg.maxRows); - tjsonAddIntegerToObject(item, "precision", pObj->cfg.precision); - tjsonAddIntegerToObject(item, "compression", pObj->cfg.compression); - tjsonAddIntegerToObject(item, "replications", pObj->cfg.replications); - tjsonAddIntegerToObject(item, "strict", pObj->cfg.strict); - tjsonAddIntegerToObject(item, "cacheLast", pObj->cfg.cacheLast); - tjsonAddIntegerToObject(item, "hashMethod", pObj->cfg.hashMethod); - tjsonAddIntegerToObject(item, "numOfRetensions", pObj->cfg.numOfRetensions); - tjsonAddIntegerToObject(item, "schemaless", pObj->cfg.schemaless); - tjsonAddIntegerToObject(item, "walLevel", pObj->cfg.walLevel); - tjsonAddIntegerToObject(item, "walFsyncPeriod", pObj->cfg.walFsyncPeriod); - tjsonAddIntegerToObject(item, "walRetentionPeriod", pObj->cfg.walRetentionPeriod); - tjsonAddIntegerToObject(item, "walRetentionSize", pObj->cfg.walRetentionSize); - tjsonAddIntegerToObject(item, "walRollPeriod", pObj->cfg.walRollPeriod); - tjsonAddIntegerToObject(item, "walSegmentSize", pObj->cfg.walSegmentSize); - - sdbRelease(pSdb, pObj); - } -} - -void dumpStb(SSdb *pSdb, SJson *json) { - void *pIter = NULL; - SJson *items = tjsonCreateObject(); - tjsonAddItemToObject(json, "stbs", items); - - while (1) { - SStbObj *pObj = NULL; - pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pObj); - if (pIter == NULL) break; - - SJson *item = tjsonCreateObject(); - tjsonAddItemToObject(items, "stb", item); - - tjsonAddStringToObject(item, "name", pObj->name); - tjsonAddStringToObject(item, "db", pObj->db); - tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); - tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); - tjsonAddStringToObject(item, "uid", i642str(pObj->uid)); - tjsonAddStringToObject(item, "dbUid", i642str(pObj->dbUid)); - tjsonAddIntegerToObject(item, "tagVer", pObj->tagVer); - tjsonAddIntegerToObject(item, "colVer", pObj->colVer); - tjsonAddIntegerToObject(item, "nextColId", pObj->nextColId); - tjsonAddIntegerToObject(item, "watermark1", pObj->watermark[0]); - tjsonAddIntegerToObject(item, "watermark2", pObj->watermark[1]); - tjsonAddIntegerToObject(item, "maxdelay1", pObj->maxdelay[0]); - tjsonAddIntegerToObject(item, "maxdelay2", pObj->maxdelay[1]); - tjsonAddIntegerToObject(item, "ttl", pObj->ttl); - tjsonAddIntegerToObject(item, "numOfColumns", pObj->numOfColumns); - tjsonAddIntegerToObject(item, "numOfTags", pObj->numOfTags); - tjsonAddIntegerToObject(item, "commentLen", pObj->commentLen); - tjsonAddIntegerToObject(item, "ast1Len", pObj->ast1Len); - tjsonAddIntegerToObject(item, "ast2Len", pObj->ast2Len); - - sdbRelease(pSdb, pObj); - } -} - -void dumpSma(SSdb *pSdb, SJson *json) {} - -void dumpVgroup(SSdb *pSdb, SJson *json) {} - -void dumpTopic(SSdb *pSdb, SJson *json) {} - -void dumpConsumber(SSdb *pSdb, SJson *json) {} - -void dumpSubscribe(SSdb *pSdb, SJson *json) {} - -void dumpOffset(SSdb *pSdb, SJson *json) {} - -void dumpStream(SSdb *pSdb, SJson *json) {} - -void dumpAcct(SSdb *pSdb, SJson *json) { - void *pIter = NULL; - SJson *items = tjsonCreateObject(); - tjsonAddItemToObject(json, "accts", items); - - while (1) { - SAcctObj *pObj = NULL; - pIter = sdbFetch(pSdb, SDB_ACCT, pIter, (void **)&pObj); - if (pIter == NULL) break; - - SJson *item = tjsonCreateObject(); - tjsonAddItemToObject(items, "acct", item); - - tjsonAddStringToObject(item, "acct", pObj->acct); - tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); - tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); - tjsonAddIntegerToObject(item, "acctId", pObj->acctId); - - sdbRelease(pSdb, pObj); - } -} - -void dumpAuth(SSdb *pSdb, SJson *json) {} - -void dumpUser(SSdb *pSdb, SJson *json) { - void *pIter = NULL; - SJson *items = tjsonCreateObject(); - tjsonAddItemToObject(json, "users", items); - - while (1) { - SUserObj *pObj = NULL; - pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pObj); - if (pIter == NULL) break; - - SJson *item = tjsonCreateObject(); - tjsonAddItemToObject(items, "user", item); - - tjsonAddStringToObject(item, "name", pObj->user); - tjsonAddStringToObject(item, "pass", pObj->pass); - tjsonAddStringToObject(item, "acct", pObj->acct); - tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); - tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); - tjsonAddIntegerToObject(item, "superUser", pObj->superUser); - tjsonAddIntegerToObject(item, "authVersion", pObj->authVersion); - tjsonAddIntegerToObject(item, "numOfReadDbs", taosHashGetSize(pObj->readDbs)); - tjsonAddIntegerToObject(item, "numOfWriteDbs", taosHashGetSize(pObj->writeDbs)); - - sdbRelease(pSdb, pObj); - } -} - -void dumpDnode(SSdb *pSdb, SJson *json) { - void *pIter = NULL; - SJson *items = tjsonCreateObject(); - tjsonAddItemToObject(json, "dnodes", items); - - while (1) { - SDnodeObj *pObj = NULL; - pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj); - if (pIter == NULL) break; - - SJson *item = tjsonCreateObject(); - tjsonAddItemToObject(items, "dnode", item); - - tjsonAddIntegerToObject(item, "id", pObj->id); - tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); - tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); - tjsonAddIntegerToObject(item, "port", pObj->port); - tjsonAddStringToObject(item, "fqdn", pObj->fqdn); - - sdbRelease(pSdb, pObj); - } -} - -void dumpBnode(SSdb *pSdb, SJson *json) {} - -void dumpSnode(SSdb *pSdb, SJson *json) {} - -void dumpQnode(SSdb *pSdb, SJson *json) {} - -void dumpMnode(SSdb *pSdb, SJson *json) { - void *pIter = NULL; - SJson *items = tjsonCreateObject(); - tjsonAddItemToObject(json, "mnodes", items); - - while (1) { - SMnodeObj *pObj = NULL; - pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj); - if (pIter == NULL) break; - - SJson *item = tjsonCreateObject(); - tjsonAddItemToObject(items, "mnode", item); - - tjsonAddIntegerToObject(item, "id", pObj->id); - tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); - tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); - - sdbRelease(pSdb, pObj); - } -} - -void dumpCluster(SSdb *pSdb, SJson *json) { - void *pIter = NULL; - SJson *items = tjsonCreateObject(); - tjsonAddItemToObject(json, "clusters", items); - - while (1) { - SClusterObj *pObj = NULL; - pIter = sdbFetch(pSdb, SDB_CLUSTER, pIter, (void **)&pObj); - if (pIter == NULL) break; - - SJson *item = tjsonCreateObject(); - tjsonAddItemToObject(items, "cluster", item); - - tjsonAddStringToObject(item, "id", i642str(pObj->id)); - tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); - tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); - tjsonAddStringToObject(item, "name", pObj->name); - - sdbRelease(pSdb, pObj); - } -} - -void dumpTrans(SSdb *pSdb, SJson *json) { - void *pIter = NULL; - SJson *items = tjsonCreateObject(); - tjsonAddItemToObject(json, "transactions", items); - - while (1) { - STrans *pObj = NULL; - pIter = sdbFetch(pSdb, SDB_TRANS, pIter, (void **)&pObj); - if (pIter == NULL) break; - - SJson *item = tjsonCreateObject(); - tjsonAddItemToObject(items, "trans", item); - - tjsonAddIntegerToObject(item, "id", pObj->id); - tjsonAddIntegerToObject(item, "stage", pObj->stage); - tjsonAddIntegerToObject(item, "policy", pObj->policy); - tjsonAddIntegerToObject(item, "conflict", pObj->conflict); - tjsonAddIntegerToObject(item, "exec", pObj->exec); - tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); - tjsonAddStringToObject(item, "dbname1", pObj->dbname1); - tjsonAddStringToObject(item, "dbname2", pObj->dbname2); - tjsonAddIntegerToObject(item, "commitLogNum", taosArrayGetSize(pObj->commitActions)); - tjsonAddIntegerToObject(item, "redoActionNum", taosArrayGetSize(pObj->redoActions)); - tjsonAddIntegerToObject(item, "undoActionNum", taosArrayGetSize(pObj->undoActions)); - - sdbRelease(pSdb, pObj); - } -} - -void dumpHeader(SSdb *pSdb, SJson *json) { - tjsonAddIntegerToObject(json, "sver", 1); - tjsonAddStringToObject(json, "applyIndex", i642str(pSdb->applyIndex)); - tjsonAddStringToObject(json, "applyTerm", i642str(pSdb->applyTerm)); - tjsonAddStringToObject(json, "applyConfig", i642str(pSdb->applyConfig)); - - SJson *maxIdsJson = tjsonCreateObject(); - tjsonAddItemToObject(json, "maxIds", maxIdsJson); - for (int32_t i = 0; i < SDB_MAX; ++i) { - int64_t maxId = 0; - if (i < SDB_MAX) { - maxId = pSdb->maxId[i]; - } - tjsonAddStringToObject(maxIdsJson, sdbTableName(i), i642str(maxId)); - } - - SJson *tableVersJson = tjsonCreateObject(); - tjsonAddItemToObject(json, "tableVers", tableVersJson); - for (int32_t i = 0; i < SDB_MAX; ++i) { - int64_t tableVer = 0; - if (i < SDB_MAX) { - tableVer = pSdb->tableVer[i]; - } - tjsonAddStringToObject(tableVersJson, sdbTableName(i), i642str(tableVer)); - } -} - -int32_t dumpSdb() { - wDebugFlag = 0; - mDebugFlag = 0; - sDebugFlag = 0; - - SMsgCb msgCb = {0}; - msgCb.reportStartupFp = reportStartup; - msgCb.sendReqFp = sendReq; - msgCb.sendRspFp = sendRsp; - msgCb.mgmt = (SMgmtWrapper *)(&msgCb); // hack - tmsgSetDefault(&msgCb); - walInit(); - syncInit(); - - SMnodeOpt opt = {.msgCb = msgCb}; - SMnode *pMnode = mndOpen(TMP_MNODE_DIR, &opt); - if (pMnode == NULL) return -1; - - SSdb *pSdb = pMnode->pSdb; - SJson *json = tjsonCreateObject(); - dumpHeader(pSdb, json); - dumpFunc(pSdb, json); - dumpDb(pSdb, json); - dumpStb(pSdb, json); - dumpSma(pSdb, json); - dumpVgroup(pSdb, json); - dumpTopic(pSdb, json); - dumpConsumber(pSdb, json); - dumpSubscribe(pSdb, json); - dumpOffset(pSdb, json); - dumpStream(pSdb, json); - dumpAcct(pSdb, json); - dumpAuth(pSdb, json); - dumpUser(pSdb, json); - dumpDnode(pSdb, json); - dumpBnode(pSdb, json); - dumpSnode(pSdb, json); - dumpQnode(pSdb, json); - dumpMnode(pSdb, json); - dumpCluster(pSdb, json); - dumpTrans(pSdb, json); - - char *pCont = tjsonToString(json); - int32_t contLen = strlen(pCont); - char file[] = "sdb.json"; - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to write %s since %s", file, terrstr()); - return -1; - } - taosWriteFile(pFile, pCont, contLen); - taosWriteFile(pFile, "\n", 1); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - tjsonDelete(json); - taosMemoryFree(pCont); - taosRemoveDir(TMP_DNODE_DIR); - return 0; -} - -int32_t parseArgs(int32_t argc, char *argv[]) { - for (int32_t i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { - if (strlen(argv[++i]) >= PATH_MAX) { - printf("config file path overflow"); - return -1; - } - tstrncpy(configDir, argv[i], PATH_MAX); - } else { - printf("'-c' requires a parameter, default is %s\n", configDir); - return -1; - } - } else { - printf("-c Configuration directory. \n"); - return -1; - } - } - - if (taosCreateLog("dumplog", 1, configDir, NULL, NULL, NULL, NULL, 1) != 0) { - printf("failed to dump since init log error\n"); - return -1; - } - - if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) { - printf("failed to dump since read config error\n"); - return -1; - } - - char mnodeJson[PATH_MAX] = {0}; - char dataFile[PATH_MAX] = {0}; - char raftCfgFile[PATH_MAX] = {0}; - char raftStoreFile[PATH_MAX] = {0}; - snprintf(mnodeJson, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "mnode.json", tsDataDir); - snprintf(dataFile, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "data" TD_DIRSEP "sdb.data", tsDataDir); - snprintf(raftCfgFile, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_config.json", tsDataDir); - snprintf(raftStoreFile, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_store.json", tsDataDir); - - char cmd[PATH_MAX * 2] = {0}; - snprintf(cmd, sizeof(cmd), "rm -rf %s", TMP_DNODE_DIR); - - system(cmd); -#ifdef WINDOWS - taosMulMkDir(TMP_SDB_DATA_DIR); - taosMulMkDir(TMP_SDB_SYNC_DIR); - snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", mnodeJson, TMP_SDB_MNODE_JSON); - system(cmd); - snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", dataFile, TMP_SDB_DATA_FILE); - system(cmd); - snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", raftCfgFile, TMP_SDB_RAFT_CFG_FILE); - system(cmd); - snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", raftStoreFile, TMP_SDB_RAFT_STORE_FILE); - system(cmd); -#else - snprintf(cmd, sizeof(cmd), "mkdir -p %s", TMP_SDB_DATA_DIR); - system(cmd); - snprintf(cmd, sizeof(cmd), "mkdir -p %s", TMP_SDB_SYNC_DIR); - system(cmd); - snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", mnodeJson, TMP_SDB_MNODE_JSON); - system(cmd); - snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", dataFile, TMP_SDB_DATA_FILE); - system(cmd); - snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", raftCfgFile, TMP_SDB_RAFT_CFG_FILE); - system(cmd); - snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", raftStoreFile, TMP_SDB_RAFT_STORE_FILE); - system(cmd); -#endif - - strcpy(tsDataDir, TMP_DNODE_DIR); - return 0; -} - -int32_t main(int32_t argc, char *argv[]) { - if (parseArgs(argc, argv) != 0) { - return -1; - } - - return dumpSdb(); -} - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/tests/test/c/sml_test.c b/tests/test/c/sml_test.c deleted file mode 100644 index 1fd1def263..0000000000 --- a/tests/test/c/sml_test.c +++ /dev/null @@ -1,1191 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include -#include -#include -#include "taos.h" -#include "types.h" - -int smlProcess_influx_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 " - "load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=" - "124,velocity=0,heading=221,grade=0 1451606401000000000", - "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 " - "load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=" - "124,velocity=0,heading=221,grade=0,fuel_consumption=25 1451607402000000000", - "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 " - "load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=" - "124,heading=221,grade=0,fuel_consumption=25 1451608403000000000", - "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 " - "fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0," - "heading=221,grade=0,fuel_consumption=25 1451609404000000000", - "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 fuel_consumption=25,grade=0 " - "1451619405000000000", - "readings,name=truck_1,fleet=South,driver=Albert,model=F-150,device_version=v1.5 " - "load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=72.45258,longitude=68.83761,elevation=" - "255,velocity=0,heading=181,grade=0,fuel_consumption=25 1451606406000000000", - "readings,name=truck_2,driver=Derek,model=F-150,device_version=v1.5 " - "load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=" - "428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451606407000000000", - "readings,name=truck_2,fleet=North,driver=Derek,model=F-150 " - "load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=" - "428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451609408000000000", - "readings,fleet=South,name=truck_0,driver=Trish,model=H-2,device_version=v2.3 fuel_consumption=25,grade=0 " - "1451629409000000000", - "stable,t1=t1,t2=t2,t3=t3 c1=1,c2=2,c3=\"kk\",c4=4 1451629501000000000", - "stable,t2=t2,t1=t1,t3=t3 c1=1,c3=\"\",c4=4 1451629602000000000", - }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - - return code; -} - -int smlProcess_telnet_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = {"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", - "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ", - "sys.if.bytes.out 1479496102 1.3E3 network=tcp", - " sys.procs.running 1479496100 42 host=web01 "}; - - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, - TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - return code; -} - -int smlProcess_json1_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "[" - " {" - " \"metric\": \"sys.cpu.nice\"," - " \"timestamp\": 0," - " \"value\": 18," - " \"tags\": {" - " \"host\": \"web01\"," - " \"id\": \"t1\"," - " \"dc\": \"lga\"" - " }" - " }," - " {" - " \"metric\": \"sys.cpu.nice\"," - " \"timestamp\": 1346846400," - " \"value\": 9," - " \"tags\": {" - " \"host\": \"web02\"," - " \"dc\": \"lga\"" - " }" - " }" - "]"}; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, - TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - return code; -} - -int smlProcess_json2_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "{" - " \"metric\": \"meter_current0\"," - " \"timestamp\": {" - " \"value\" : 1346846400," - " \"type\" : \"s\"" - " }," - " \"value\": {" - " \"value\" : 10.3," - " \"type\" : \"i64\"" - " }," - " \"tags\": {" - " \"groupid\": { " - " \"value\" : 2," - " \"type\" : \"bigint\"" - " }," - " \"location\": { " - " \"value\" : \"北京\"," - " \"type\" : \"binary\"" - " }," - " \"id\": \"d1001\"" - " }" - "}"}; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, - TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - return code; -} - -int smlProcess_json3_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "{" - " \"metric\": \"meter_current1\"," - " \"timestamp\": {" - " \"value\" : 1346846400," - " \"type\" : \"s\"" - " }," - " \"value\": {" - " \"value\" : 10.3," - " \"type\" : \"i64\"" - " }," - " \"tags\": {" - " \"t1\": { " - " \"value\" : 2," - " \"type\" : \"bigint\"" - " }," - " \"t2\": { " - " \"value\" : 2," - " \"type\" : \"int\"" - " }," - " \"t3\": { " - " \"value\" : 2," - " \"type\" : \"i16\"" - " }," - " \"t4\": { " - " \"value\" : 2," - " \"type\" : \"i8\"" - " }," - " \"t5\": { " - " \"value\" : 2," - " \"type\" : \"f32\"" - " }," - " \"t6\": { " - " \"value\" : 2," - " \"type\" : \"double\"" - " }," - " \"t7\": { " - " \"value\" : \"8323\"," - " \"type\" : \"binary\"" - " }," - " \"t8\": { " - " \"value\" : \"北京\"," - " \"type\" : \"nchar\"" - " }," - " \"t9\": { " - " \"value\" : true," - " \"type\" : \"bool\"" - " }," - " \"id\": \"d1001\"" - " }" - "}"}; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, - TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - return code; -} - -int smlProcess_json4_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "{" - " \"metric\": \"meter_current2\"," - " \"timestamp\": {" - " \"value\" : 1346846500000," - " \"type\" : \"ms\"" - " }," - " \"value\": \"ni\"," - " \"tags\": {" - " \"t1\": { " - " \"value\" : 20," - " \"type\" : \"i64\"" - " }," - " \"t2\": { " - " \"value\" : 25," - " \"type\" : \"i32\"" - " }," - " \"t3\": { " - " \"value\" : 2," - " \"type\" : \"smallint\"" - " }," - " \"t4\": { " - " \"value\" : 2," - " \"type\" : \"tinyint\"" - " }," - " \"t5\": { " - " \"value\" : 2," - " \"type\" : \"float\"" - " }," - " \"t6\": { " - " \"value\" : 0.2," - " \"type\" : \"f64\"" - " }," - " \"t7\": \"nsj\"," - " \"t8\": { " - " \"value\" : \"北京\"," - " \"type\" : \"nchar\"" - " }," - " \"t9\": false," - " \"id\": \"d1001\"" - " }" - "}"}; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, - TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - return code; -} - -int sml_TD15662_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db precision 'ns' schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "hetrey c0=f,c1=127i8 1626006833639", - "hetrey,t1=r c0=f,c1=127i8 1626006833640", - }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, - TSDB_SML_TIMESTAMP_MILLI_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - return code; -} - -int sml_TD15742_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "test_ms,t0=t c0=f 1626006833641", - }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, - TSDB_SML_TIMESTAMP_MILLI_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - return code; -} - -int sml_16384_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - const char *sql[] = { - "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=t,c1=127i8 1626006833639000000", - }; - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - pRes = taos_schemaless_insert(taos, (char **)sql, 1, TSDB_SML_LINE_PROTOCOL, 0); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - if(code) return code; - - const char *sql1[] = { - "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=f,c1=127i8,c11=L\"ncharColValue\",c10=t 1626006833639000000", - }; - pRes = taos_schemaless_insert(taos, (char **)sql1, 1, TSDB_SML_LINE_PROTOCOL, 0); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - code = taos_errno(pRes); - taos_free_result(pRes); - return code; -} - -int sml_oom_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - const char *sql[] = { - //"test_ms,t0=t c0=f 1626006833641", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"pgxbrbga\",t8=L\"ncharTagValue\" " - "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"gviggpmi\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"cexkarjn\",t8=L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"rzwwuoxu\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"xphrlkey\",t8=L\"ncharTagValue\" " - "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"llsawebj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"jwpkipff\",t8=L\"ncharTagValue\" " - "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"euzzhcvu\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"jumhnsvw\",t8=L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"fnetgdhj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"vrmmpgqe\",t8=L\"ncharTagValue\" " - "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"lnpfjapr\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"gvbhmsfr\",t8=L\"ncharTagValue\" " - "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"kydxrxwc\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"pfyarryq\",t8=L\"ncharTagValue\" " - "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"uxptotap\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"prolhudh\",t8=L\"ncharTagValue\" " - "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"ttxaxnac\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"dfgvmjmz\",t8=L\"ncharTagValue\" " - "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"bloextkn\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"dvjxwzsi\",t8=L\"ncharTagValue\" " - "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"aigjomaf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"refbidtf\",t8=L\"ncharTagValue\" " - "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"vuanlfpz\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"nbpajxkx\",t8=L\"ncharTagValue\" " - "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"ktzzauxh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"prcwdjct\",t8=L\"ncharTagValue\" " - "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"vmbhvjtp\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"liuddtuz\",t8=L\"ncharTagValue\" " - "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"pddsktow\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"algldlvl\",t8=L\"ncharTagValue\" " - "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"mlmnjgdl\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"oiynpcog\",t8=L\"ncharTagValue\" " - "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"wmynbagb\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"asvyulrm\",t8=L\"ncharTagValue\" " - "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"ohaacrkp\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"ytyejhiq\",t8=L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"bbznuerb\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"lpebcibw\",t8=L\"ncharTagValue\" " - "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"xmqrbafv\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"lnmwpdne\",t8=L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"jpcsjqun\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"mmxqmavz\",t8=L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"hhsbgaow\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"uwogyuud\",t8=L\"ncharTagValue\" " - "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"ytxpaxnk\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"wouwdvtt\",t8=L\"ncharTagValue\" " - "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"iitwikkh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"lgyzuyaq\",t8=L\"ncharTagValue\" " - "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"bdtiigxi\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"qpnsvdhw\",t8=L\"ncharTagValue\" " - "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"pjxihgvu\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"ksxkfetn\",t8=L\"ncharTagValue\" " - "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"ocukufqs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"qzerxmpe\",t8=L\"ncharTagValue\" " - "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"qwcfdyxs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"jldrpmmd\",t8=L\"ncharTagValue\" " - "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"lucxlfzc\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"rcewrvya\",t8=L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"dknvaphs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"nxtxgzdr\",t8=L\"ncharTagValue\" " - "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"mbvuugwz\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"uikakffu\",t8=L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"mwmtqsma\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"bfcxrrpa\",t8=L\"ncharTagValue\" " - "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"ksajygdj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"vmhhszyv\",t8=L\"ncharTagValue\" " - "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"urwjgvut\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"jrvytcxy\",t8=L\"ncharTagValue\" " - "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"evqkzygh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"zitdznhg\",t8=L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"tpqekrxa\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"yrrbgjtk\",t8=L\"ncharTagValue\" " - "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"bnphiuyq\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"huknehjn\",t8=L\"ncharTagValue\" " - "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"iudbxfke\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"fjmolwbn\",t8=L\"ncharTagValue\" " - "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"gukzgcjs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"bjvdtlgq\",t8=L\"ncharTagValue\" " - "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"phxnesxh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"qgpgckvc\",t8=L\"ncharTagValue\" " - "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"yechqtfa\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"pbouxywy\",t8=L\"ncharTagValue\" " - "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"kxtuojyo\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"txaniwlj\",t8=L\"ncharTagValue\" " - "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"fixgufrj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"okzvalwq\",t8=L\"ncharTagValue\" " - "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"iitawgbn\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"gayvmird\",t8=L\"ncharTagValue\" " - "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"dprkfjph\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"kmuccshq\",t8=L\"ncharTagValue\" " - "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"vkslsdsd\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"dukccdqk\",t8=L\"ncharTagValue\" " - "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"leztxmqf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"kltixbwz\",t8=L\"ncharTagValue\" " - "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"xqhkweef\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"idxsimvz\",t8=L\"ncharTagValue\" " - "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"vbruvcpk\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"uxandqkd\",t8=L\"ncharTagValue\" " - "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"dsiosysh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"kxuyanpp\",t8=L\"ncharTagValue\" " - "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"wkrktags\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"yvizzpiv\",t8=L\"ncharTagValue\" " - "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"ddnefben\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"novmfmbc\",t8=L\"ncharTagValue\" " - "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"fnusxsfu\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"ouerfjap\",t8=L\"ncharTagValue\" " - "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"sigognkf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"slvzhede\",t8=L\"ncharTagValue\" " - "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"bknerect\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"tmhcdfjb\",t8=L\"ncharTagValue\" " - "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"hpnoanpp\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"okmhelnc\",t8=L\"ncharTagValue\" " - "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"xcernjin\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"jdmiismg\",t8=L\"ncharTagValue\" " - "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"tmnqozrf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"zgwrftkx\",t8=L\"ncharTagValue\" " - "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"zyamlwwh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"nuedqcro\",t8=L\"ncharTagValue\" " - "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"lpsvyqaa\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"mneitsul\",t8=L\"ncharTagValue\" " - "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"vpleinwb\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"njxuaedy\",t8=L\"ncharTagValue\" " - "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"sdgxpqmu\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"yjirrebp\",t8=L\"ncharTagValue\" " - "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"ikqndzfj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"ghnfdxhr\",t8=L\"ncharTagValue\" " - "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"hrwczpvo\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"nattumpb\",t8=L\"ncharTagValue\" " - "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"zoyfzazn\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"rdwemofy\",t8=L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"phkgsjeg\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"pyhvvjrt\",t8=L\"ncharTagValue\" " - "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"zfslyton\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"bxwjzeri\",t8=L\"ncharTagValue\" " - "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"uovzzgjv\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"cfjmacvr\",t8=L\"ncharTagValue\" " - "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"jefqgzqx\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"njrksxmr\",t8=L\"ncharTagValue\" " - "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"mhvabvgn\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"kfekjltr\",t8=L\"ncharTagValue\" " - "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"lexfaaby\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"zbblsmwq\",t8=L\"ncharTagValue\" " - "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"oqcombkx\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"rcdmhzyw\",t8=L\"ncharTagValue\" " - "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"otksuean\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"itbdvowq\",t8=L\"ncharTagValue\" " - "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"tswtmhex\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"xoukkzid\",t8=L\"ncharTagValue\" " - "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"guangmpq\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"rayxzuky\",t8=L\"ncharTagValue\" " - "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"lspwucrv\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"pdprzzkf\",t8=L\"ncharTagValue\" " - "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"sddqrtza\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"kabndgkx\",t8=L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"aglnqqxs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"fiwpzmdr\",t8=L\"ncharTagValue\" " - "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"hxctooen\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"pckjpwyh\",t8=L\"ncharTagValue\" " - "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"ivmvsbai\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"eljdclst\",t8=L\"ncharTagValue\" " - "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"rwgdctie\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"zlnthxoz\",t8=L\"ncharTagValue\" " - "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"ljtxelle\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"llfggdpy\",t8=L\"ncharTagValue\" " - "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"tvnridze\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"hxjpgube\",t8=L\"ncharTagValue\" " - "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"zmldmquq\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"bggqwcoj\",t8=L\"ncharTagValue\" " - "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"drksfofm\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"jcsixens\",t8=L\"ncharTagValue\" " - "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"cdwnwhaf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"nngpumuq\",t8=L\"ncharTagValue\" " - "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"hylgooci\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"cozeyjys\",t8=L\"ncharTagValue\" " - "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"lcgpfcsa\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"qdtzhtyd\",t8=L\"ncharTagValue\" " - "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"txpubynb\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"gbslzbtu\",t8=L\"ncharTagValue\" " - "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"buihcpcl\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"ayqezaiq\",t8=L\"ncharTagValue\" " - "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"zgkgtilj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"bcjopqif\",t8=L\"ncharTagValue\" " - "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"mfzxiaqt\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"xmnlqxoj\",t8=L\"ncharTagValue\" " - "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"reyiklyf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"xssuomhk\",t8=L\"ncharTagValue\" " - "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"liazkjll\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"nigjlblo\",t8=L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"vmojyznk\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"dotkbvrz\",t8=L\"ncharTagValue\" " - "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"kuwdyydw\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"slsfqydw\",t8=L\"ncharTagValue\" " - "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"zyironhd\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"pktwfhzi\",t8=L\"ncharTagValue\" " - "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"xybavsvh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"pyrxemvx\",t8=L\"ncharTagValue\" " - "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"tlfihwjs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," - "t7=\"neumakmg\",t8=L\"ncharTagValue\" " - "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"wxqingoa\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - }; - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - return code; -} - -int sml_16368_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "[{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639000, \"type\": \"us\"}, \"value\": 1, " - "\"tags\": {\"t1\": 3, \"t2\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t3\", \"type\": " - "\"binary\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833739000, \"type\": \"us\"}, \"value\": 2, " - "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639100, \"type\": \"us\"}, \"value\": 3, " - "\"tags\": {\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t3\": {\"value\": \"ste\", \"type\": \"nchar\"}}}," - "{\"metric\": \"stf567890\", \"timestamp\": {\"value\": 1626006833639200, \"type\": \"us\"}, \"value\": 4, " - "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"bigint\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639300, \"type\": \"us\"}, \"value\": " - "{\"value\": 5, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t2\": 5.0, " - "\"t3\": {\"value\": \"t4\", \"type\": \"binary\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639400, \"type\": \"us\"}, \"value\": " - "{\"value\": 6, \"type\": \"double\"}, \"tags\": {\"t2\": 5.0, \"t3\": {\"value\": \"ste2\", \"type\": " - "\"nchar\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006834639400, \"type\": \"us\"}, \"value\": " - "{\"value\": 7, \"type\": \"double\"}, \"tags\": {\"t2\": {\"value\": 5.0, \"type\": \"double\"}, \"t3\": " - "{\"value\": \"ste2\", \"type\": \"nchar\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833839006, \"type\": \"us\"}, \"value\": " - "{\"value\": 8, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": " - "{\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, " - "\"type\": \"double\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833939007, \"type\": \"us\"}, \"value\": " - "{\"value\": 9, \"type\": \"double\"}, \"tags\": {\"t1\": 4, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}]"}; - pRes = taos_schemaless_insert(taos, (char **)sql, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - return code; -} - -int sml_dup_time_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - const char *sql[] = {//"test_ms,t0=t c0=f 1626006833641", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=false,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006834639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006835639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006836639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006837639000000"}; - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - return code; -} - -int sml_16960_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "[" - "{" - "\"timestamp\":" - "" - "{ \"value\": 1349020800000, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": 830525384, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1349020800001, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -588348364, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1349020800002, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -370310823, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1349020800003, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -811250191, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1349020800004, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -330340558, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}" - "]"}; - - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, - TSDB_SML_TIMESTAMP_MILLI_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - return code; -} - -int sml_add_tag_col_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - const char *sql[] = { - "macylr,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000" - }; - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - if (code) return code; - - const char *sql1[] = { - "macylr,id=macylr_17875_1804,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\",t11=127i8,t10=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c8=L\"ncharColValue\",c9=7u64,c11=L\"ncharColValue\",c10=f 1626006833639000000" - }; - - pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, 0); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - code = taos_errno(pRes); - taos_free_result(pRes); - - return code; -} - -int smlProcess_18784_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "disk,device=sdc inodes_used=176059i,total=1081101176832i 1661943960000000000", - "disk,device=sdc inodes_free=66932805i 1661943960000000000", - }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); - printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); - int code = taos_errno(pRes); - ASSERT(!code); - ASSERT(taos_affected_rows(pRes) == 2); - taos_free_result(pRes); - - pRes = taos_query(taos, "select * from disk"); - ASSERT(pRes); - int fieldNum = taos_field_count(pRes); - ASSERT(fieldNum == 5); - printf("fieldNum:%d\n", fieldNum); - TAOS_ROW row = NULL; - int32_t rowIndex = 0; - while((row = taos_fetch_row(pRes)) != NULL) { - int64_t ts = *(int64_t*)row[0]; - int64_t used = *(int64_t*)row[1]; - int64_t total = *(int64_t*)row[2]; - int64_t freed = *(int64_t*)row[3]; - if(rowIndex == 0){ - ASSERT(ts == 1661943960000); - ASSERT(used == 176059); - ASSERT(total == 1081101176832); - ASSERT(freed == 66932805); -// ASSERT_EQ(latitude, 24.5208); -// ASSERT_EQ(longitude, 28.09377); -// ASSERT_EQ(elevation, 428); -// ASSERT_EQ(velocity, 0); -// ASSERT_EQ(heading, 304); -// ASSERT_EQ(grade, 0); -// ASSERT_EQ(fuel_consumption, 25); - }else{ -// ASSERT(0); - } - rowIndex++; - } - taos_free_result(pRes); - - return code; -} - -int main(int argc, char *argv[]) { - int ret = 0; - ret = smlProcess_influx_Test(); - ASSERT(!ret); - ret = smlProcess_telnet_Test(); - ASSERT(!ret); - ret = smlProcess_json1_Test(); - ASSERT(!ret); - ret = smlProcess_json2_Test(); - ASSERT(!ret); - ret = smlProcess_json3_Test(); - ASSERT(!ret); - ret = smlProcess_json4_Test(); - ASSERT(!ret); - ret = sml_TD15662_Test(); - ASSERT(!ret); - ret = sml_TD15742_Test(); - ASSERT(!ret); - ret = sml_16384_Test(); - ASSERT(!ret); - ret = sml_oom_Test(); - ASSERT(!ret); - ret = sml_16368_Test(); - ASSERT(!ret); - ret = sml_dup_time_Test(); - ASSERT(!ret); - ret = sml_16960_Test(); - ASSERT(!ret); - ret = sml_add_tag_col_Test(); - ASSERT(!ret); - ret = smlProcess_18784_Test(); - ASSERT(!ret); - return ret; -} diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c deleted file mode 100644 index 784b45c92b..0000000000 --- a/tests/test/c/tmqDemo.c +++ /dev/null @@ -1,705 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include -#include -#include -#include -#include -// #include - -#include "taos.h" -#include "taoserror.h" -#include "tlog.h" - -#define GREEN "\033[1;32m" -#define NC "\033[0m" -#define min(a, b) (((a) < (b)) ? (a) : (b)) - -#define MAX_SQL_STR_LEN (1024 * 1024) -#define MAX_ROW_STR_LEN (16 * 1024) - -enum _RUN_MODE { - TMQ_RUN_INSERT_AND_CONSUME, - TMQ_RUN_ONLY_INSERT, - TMQ_RUN_ONLY_CONSUME, - TMQ_RUN_MODE_BUTT, -}; - -typedef struct { - char dbName[32]; - char stbName[64]; - char resultFileName[256]; - char vnodeWalPath[256]; - int32_t numOfThreads; - int32_t numOfTables; - int32_t numOfVgroups; - int32_t runMode; - int32_t numOfColumn; - double ratio; - int32_t batchNumOfRow; - int32_t totalRowsOfPerTbl; - int64_t startTimestamp; - int32_t showMsgFlag; - int32_t simCase; - - int32_t totalRowsOfT2; -} SConfInfo; - -static SConfInfo g_stConfInfo = { - "tmqdb", - "stb", - "./tmqResult.txt", // output_file - "", // /data2/dnode/data/vnode/vnode2/wal", - 1, // threads - 1, // tables - 1, // vgroups - 0, // run mode - 1, // columns - 1, // ratio - 1, // batch size - 10000, // total rows for per table - 0, // 2020-01-01 00:00:00.000 - 0, // show consume msg switch - 0, // if run in sim case - 10000, -}; - -char* g_pRowValue = NULL; -TdFilePtr g_fp = NULL; - -static void printHelp() { - char indent[10] = " "; - printf("Used to test the performance while create table\n"); - - printf("%s%s\n", indent, "-c"); - printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); - printf("%s%s\n", indent, "-d"); - printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", g_stConfInfo.dbName); - printf("%s%s\n", indent, "-s"); - printf("%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", g_stConfInfo.stbName); - printf("%s%s\n", indent, "-f"); - printf("%s%s%s%s\n", indent, indent, "The file of result, default is ", g_stConfInfo.resultFileName); - printf("%s%s\n", indent, "-w"); - printf("%s%s%s%s\n", indent, indent, "The path of vnode of wal, default is ", g_stConfInfo.vnodeWalPath); - printf("%s%s\n", indent, "-t"); - printf("%s%s%s%d\n", indent, indent, "numOfThreads, default is ", g_stConfInfo.numOfThreads); - printf("%s%s\n", indent, "-n"); - printf("%s%s%s%d\n", indent, indent, "numOfTables, default is ", g_stConfInfo.numOfTables); - printf("%s%s\n", indent, "-v"); - printf("%s%s%s%d\n", indent, indent, "numOfVgroups, default is ", g_stConfInfo.numOfVgroups); - printf("%s%s\n", indent, "-a"); - printf("%s%s%s%d\n", indent, indent, "runMode, default is ", g_stConfInfo.runMode); - printf("%s%s\n", indent, "-l"); - printf("%s%s%s%d\n", indent, indent, "numOfColumn, default is ", g_stConfInfo.numOfColumn); - printf("%s%s\n", indent, "-q"); - printf("%s%s%s%f\n", indent, indent, "ratio, default is ", g_stConfInfo.ratio); - printf("%s%s\n", indent, "-b"); - printf("%s%s%s%d\n", indent, indent, "batchNumOfRow, default is ", g_stConfInfo.batchNumOfRow); - printf("%s%s\n", indent, "-r"); - printf("%s%s%s%d\n", indent, indent, "totalRowsOfPerTbl, default is ", g_stConfInfo.totalRowsOfPerTbl); - printf("%s%s\n", indent, "-m"); - printf("%s%s%s%" PRId64 "\n", indent, indent, "startTimestamp, default is ", g_stConfInfo.startTimestamp); - printf("%s%s\n", indent, "-g"); - printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag); - printf("%s%s\n", indent, "-sim"); - printf("%s%s%s%d\n", indent, indent, "simCase, default is ", g_stConfInfo.simCase); - - exit(EXIT_SUCCESS); -} - -void parseArgument(int32_t argc, char* argv[]) { - g_stConfInfo.startTimestamp = 1640966400000; // 2020-01-01 00:00:00.000 - - for (int32_t i = 1; i < argc; i++) { - if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { - printHelp(); - exit(0); - } else if (strcmp(argv[i], "-d") == 0) { - strcpy(g_stConfInfo.dbName, argv[++i]); - } else if (strcmp(argv[i], "-c") == 0) { - strcpy(configDir, argv[++i]); - } else if (strcmp(argv[i], "-s") == 0) { - strcpy(g_stConfInfo.stbName, argv[++i]); - } else if (strcmp(argv[i], "-w") == 0) { - strcpy(g_stConfInfo.vnodeWalPath, argv[++i]); - } else if (strcmp(argv[i], "-f") == 0) { - strcpy(g_stConfInfo.resultFileName, argv[++i]); - } else if (strcmp(argv[i], "-t") == 0) { - g_stConfInfo.numOfThreads = atoi(argv[++i]); - } else if (strcmp(argv[i], "-n") == 0) { - g_stConfInfo.numOfTables = atoll(argv[++i]); - } else if (strcmp(argv[i], "-v") == 0) { - g_stConfInfo.numOfVgroups = atoi(argv[++i]); - } else if (strcmp(argv[i], "-a") == 0) { - g_stConfInfo.runMode = atoi(argv[++i]); - } else if (strcmp(argv[i], "-b") == 0) { - g_stConfInfo.batchNumOfRow = atoi(argv[++i]); - } else if (strcmp(argv[i], "-r") == 0) { - g_stConfInfo.totalRowsOfPerTbl = atoi(argv[++i]); - } else if (strcmp(argv[i], "-l") == 0) { - g_stConfInfo.numOfColumn = atoi(argv[++i]); - } else if (strcmp(argv[i], "-q") == 0) { - g_stConfInfo.ratio = atof(argv[++i]); - } else if (strcmp(argv[i], "-m") == 0) { - g_stConfInfo.startTimestamp = atol(argv[++i]); - } else if (strcmp(argv[i], "-g") == 0) { - g_stConfInfo.showMsgFlag = atol(argv[++i]); - } else if (strcmp(argv[i], "-sim") == 0) { - g_stConfInfo.simCase = atol(argv[++i]); - } else { - printf("%s unknow para: %s %s", GREEN, argv[++i], NC); - exit(-1); - } - } - - g_stConfInfo.totalRowsOfT2 = g_stConfInfo.totalRowsOfPerTbl * g_stConfInfo.ratio; - -#if 0 - pPrint("%s configDir:%s %s", GREEN, configDir, NC); - pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC); - pPrint("%s stbName:%s %s", GREEN, g_stConfInfo.stbName, NC); - pPrint("%s resultFileName:%s %s", GREEN, g_stConfInfo.resultFileName, NC); - pPrint("%s vnodeWalPath:%s %s", GREEN, g_stConfInfo.vnodeWalPath, NC); - pPrint("%s numOfTables:%d %s", GREEN, g_stConfInfo.numOfTables, NC); - pPrint("%s numOfThreads:%d %s", GREEN, g_stConfInfo.numOfThreads, NC); - pPrint("%s numOfVgroups:%d %s", GREEN, g_stConfInfo.numOfVgroups, NC); - pPrint("%s runMode:%d %s", GREEN, g_stConfInfo.runMode, NC); - pPrint("%s ratio:%f %s", GREEN, g_stConfInfo.ratio, NC); - pPrint("%s numOfColumn:%d %s", GREEN, g_stConfInfo.numOfColumn, NC); - pPrint("%s batchNumOfRow:%d %s", GREEN, g_stConfInfo.batchNumOfRow, NC); - pPrint("%s totalRowsOfPerTbl:%d %s", GREEN, g_stConfInfo.totalRowsOfPerTbl, NC); - pPrint("%s totalRowsOfT2:%d %s", GREEN, g_stConfInfo.totalRowsOfT2, NC); - pPrint("%s startTimestamp:%" PRId64" %s", GREEN, g_stConfInfo.startTimestamp, NC); - pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); -#endif -} - -static int running = 1; -/*static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }*/ - -// calc dir size (not include itself 4096Byte) -int64_t getDirectorySize(char* dir) { - TdDirPtr pDir; - TdDirEntryPtr pDirEntry; - int64_t totalSize = 0; - - if ((pDir = taosOpenDir(dir)) == NULL) { - fprintf(stderr, "Cannot open dir: %s\n", dir); - return -1; - } - - // lstat(dir, &statbuf); - // totalSize+=statbuf.st_size; - - while ((pDirEntry = taosReadDir(pDir)) != NULL) { - char subdir[1024]; - char* fileName = taosGetDirEntryName(pDirEntry); - sprintf(subdir, "%s/%s", dir, fileName); - - // printf("===d_name: %s\n", entry->d_name); - if (taosIsDir(subdir)) { - if (strcmp(".", fileName) == 0 || strcmp("..", fileName) == 0) { - continue; - } - - int64_t subDirSize = getDirectorySize(subdir); - totalSize += subDirSize; - } else if (0 == strcmp(strchr(fileName, '.'), ".log")) { // only calc .log file size, and not include .idx file - int64_t file_size = 0; - taosStatFile(subdir, &file_size, NULL); - totalSize += file_size; - } - } - - taosCloseDir(&pDir); - return totalSize; -} - -int queryDB(TAOS* taos, char* command) { - TAOS_RES* pRes = taos_query(taos, command); - int code = taos_errno(pRes); - if (code != 0) { - pError("failed to reason:%s, sql: %s", tstrerror(code), command); - taos_free_result(pRes); - return -1; - } - taos_free_result(pRes); - return 0; -} - -int32_t init_env() { - char sqlStr[1024] = {0}; - - TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); - if (pConn == NULL) { - return -1; - } - - sprintf(sqlStr, "create database if not exists %s vgroups %d", g_stConfInfo.dbName, g_stConfInfo.numOfVgroups); - TAOS_RES* pRes = taos_query(pConn, sqlStr); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - sprintf(sqlStr, "use %s", g_stConfInfo.dbName); - pRes = taos_query(pConn, sqlStr); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - // create row value - g_pRowValue = (char*)taosMemoryCalloc(1, g_stConfInfo.numOfColumn * 16 + 128); - if (NULL == g_pRowValue) { - return -1; - } - - int32_t dataLen = 0; - int32_t sqlLen = 0; - sqlLen += sprintf(sqlStr + sqlLen, "create stable if not exists %s (ts timestamp, ", g_stConfInfo.stbName); - for (int32_t i = 0; i < g_stConfInfo.numOfColumn; i++) { - if (i == g_stConfInfo.numOfColumn - 1) { - sqlLen += sprintf(sqlStr + sqlLen, "c%d int) ", i); - memcpy(g_pRowValue + dataLen, "66778899", strlen("66778899")); - dataLen += strlen("66778899"); - } else { - sqlLen += sprintf(sqlStr + sqlLen, "c%d int, ", i); - memcpy(g_pRowValue + dataLen, "66778899, ", strlen("66778899, ")); - dataLen += strlen("66778899, "); - } - } - sqlLen += sprintf(sqlStr + sqlLen, "tags (t0 int)"); - - pRes = taos_query(pConn, sqlStr); - if (taos_errno(pRes) != 0) { - printf("failed to create super table %s, reason:%s\n", g_stConfInfo.stbName, taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - for (int32_t i = 0; i < g_stConfInfo.numOfTables; i++) { - sprintf(sqlStr, "create table if not exists %s%d using %s tags(1)", g_stConfInfo.stbName, i, g_stConfInfo.stbName); - pRes = taos_query(pConn, sqlStr); - if (taos_errno(pRes) != 0) { - printf("failed to create child table %s%d, reason:%s\n", g_stConfInfo.stbName, i, taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - } - - // const char* sql = "select * from tu1"; - sprintf(sqlStr, "create topic test_stb_topic_1 as select ts,c0 from %s", g_stConfInfo.stbName); - /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sqlStr, strlen(sqlStr));*/ - pRes = taos_query(pConn, sqlStr); - if (taos_errno(pRes) != 0) { - printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - taos_close(pConn); - return 0; -} - -tmq_t* build_consumer() { -#if 0 - char sqlStr[1024] = {0}; - - TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - sprintf(sqlStr, "use %s", g_stConfInfo.dbName); - TAOS_RES* pRes = taos_query(pConn, sqlStr); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); -#endif - - tmq_conf_t* conf = tmq_conf_new(); - tmq_conf_set(conf, "group.id", "tg2"); - tmq_conf_set(conf, "td.connect.user", "root"); - tmq_conf_set(conf, "td.connect.pass", "taosdata"); - tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); - tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); - assert(tmq); - tmq_conf_destroy(conf); - return tmq; -} - -tmq_list_t* build_topic_list() { - tmq_list_t* topic_list = tmq_list_new(); - tmq_list_append(topic_list, "test_stb_topic_1"); - return topic_list; -} - -void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { - static const int MIN_COMMIT_COUNT = 1000; - - int msg_count = 0; - int32_t err; - - if ((err = tmq_subscribe(tmq, topics))) { - fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); - return; - } - - while (running) { - TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1); - if (tmqmessage) { - /*msg_process(tmqmessage);*/ - taos_free_result(tmqmessage); - - if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit_sync(tmq, NULL); - } - } - - err = tmq_consumer_close(tmq); - if (err) - fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); - else - fprintf(stderr, "%% Consumer closed\n"); -} - -void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLogSize) { - int32_t err; - - if ((err = tmq_subscribe(tmq, topics))) { - fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); - printf("subscribe err\n"); - return; - } - /*taosSsleep(3);*/ - int32_t batchCnt = 0; - int64_t startTime = taosGetTimestampUs(); - while (running) { - TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 3000); - if (tmqmessage) { - batchCnt++; - if (0 != g_stConfInfo.showMsgFlag) { - /*msg_process(tmqmessage);*/ - } - taos_free_result(tmqmessage); - } else { - break; - } - } - int64_t endTime = taosGetTimestampUs(); - double consumeTime = (double)(endTime - startTime) / 1000000; - - if (batchCnt != totalMsgs) { - printf("%s inserted msgs: %d and consume msgs: %d mismatch %s", GREEN, totalMsgs, batchCnt, NC); - /*exit(-1);*/ - } - - if (0 == g_stConfInfo.simCase) { - printf("consume result: msgs: %d, time used:%.3f second\n", batchCnt, consumeTime); - } else { - printf("{consume success: %d}", totalMsgs); - } - taosFprintfFile(g_fp, "|%10d | %10.3f | %8.2f | %10.2f| %10.2f |\n", batchCnt, consumeTime, - (double)batchCnt / consumeTime, (double)walLogSize / (1024 * 1024.0) / consumeTime, - (double)walLogSize / 1024.0 / batchCnt); - - err = tmq_consumer_close(tmq); - if (err) { - fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); - exit(-1); - } -} - -// sync insertion -int32_t syncWriteData() { - TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); - if (pConn == NULL) { - return -1; - } - - char sqlStr[1024] = {0}; - sprintf(sqlStr, "use %s", g_stConfInfo.dbName); - TAOS_RES* pRes = taos_query(pConn, sqlStr); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - char* buffer = NULL; - buffer = (char*)taosMemoryMalloc(MAX_SQL_STR_LEN); - if (NULL == buffer) { - return -1; - } - - int32_t totalMsgs = 0; - - int64_t time_counter = g_stConfInfo.startTimestamp; - for (int i = 0; i < g_stConfInfo.totalRowsOfPerTbl;) { - for (int tID = 0; tID <= g_stConfInfo.numOfTables - 1; tID++) { - int inserted = i; - int64_t tmp_time = time_counter; - - int32_t data_len = 0; - data_len += sprintf(buffer + data_len, "insert into %s%d values", g_stConfInfo.stbName, tID); - int k; - for (k = 0; k < g_stConfInfo.batchNumOfRow;) { - data_len += sprintf(buffer + data_len, "(%" PRId64 ", %s) ", tmp_time++, g_pRowValue); - inserted++; - k++; - - if (inserted >= g_stConfInfo.totalRowsOfPerTbl) { - break; - } - - if (data_len > MAX_SQL_STR_LEN - MAX_ROW_STR_LEN) { - break; - } - } - - int code = queryDB(pConn, buffer); - if (0 != code) { - fprintf(stderr, "insert data error!\n"); - taosMemoryFreeClear(buffer); - return -1; - } - - totalMsgs++; - - if (tID == g_stConfInfo.numOfTables - 1) { - i = inserted; - time_counter = tmp_time; - } - } - } - taosMemoryFreeClear(buffer); - return totalMsgs; -} - -// sync insertion -int32_t syncWriteDataByRatio() { - TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); - if (pConn == NULL) { - return -1; - } - - char sqlStr[1024] = {0}; - sprintf(sqlStr, "use %s", g_stConfInfo.dbName); - TAOS_RES* pRes = taos_query(pConn, sqlStr); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - char* buffer = NULL; - buffer = (char*)taosMemoryMalloc(MAX_SQL_STR_LEN); - if (NULL == buffer) { - return -1; - } - - int32_t totalMsgs = 0; - - int32_t insertedOfT1 = 0; - int32_t insertedOfT2 = 0; - - int64_t tsOfT1 = g_stConfInfo.startTimestamp; - int64_t tsOfT2 = g_stConfInfo.startTimestamp; - int64_t tmp_time; - - for (;;) { - if ((insertedOfT1 >= g_stConfInfo.totalRowsOfPerTbl) && (insertedOfT2 >= g_stConfInfo.totalRowsOfT2)) { - break; - } - - for (int tID = 0; tID <= g_stConfInfo.numOfTables - 1; tID++) { - if (0 == tID) { - tmp_time = tsOfT1; - if (insertedOfT1 >= g_stConfInfo.totalRowsOfPerTbl) { - continue; - } - } else if (1 == tID) { - tmp_time = tsOfT2; - if (insertedOfT2 >= g_stConfInfo.totalRowsOfT2) { - continue; - } - } - - int32_t data_len = 0; - data_len += sprintf(buffer + data_len, "insert into %s%d values", g_stConfInfo.stbName, tID); - int k; - for (k = 0; k < g_stConfInfo.batchNumOfRow;) { - data_len += sprintf(buffer + data_len, "(%" PRId64 ", %s) ", tmp_time++, g_pRowValue); - k++; - if (0 == tID) { - insertedOfT1++; - if (insertedOfT1 >= g_stConfInfo.totalRowsOfPerTbl) { - break; - } - } else if (1 == tID) { - insertedOfT2++; - if (insertedOfT2 >= g_stConfInfo.totalRowsOfT2) { - break; - } - } - - if (data_len > MAX_SQL_STR_LEN - MAX_ROW_STR_LEN) { - break; - } - } - - int code = queryDB(pConn, buffer); - if (0 != code) { - fprintf(stderr, "insert data error!\n"); - taosMemoryFreeClear(buffer); - return -1; - } - - if (0 == tID) { - tsOfT1 = tmp_time; - } else if (1 == tID) { - tsOfT2 = tmp_time; - } - - totalMsgs++; - } - } - pPrint("expect insert rows: T1[%d] T2[%d], actual insert rows: T1[%d] T2[%d]\n", g_stConfInfo.totalRowsOfPerTbl, - g_stConfInfo.totalRowsOfT2, insertedOfT1, insertedOfT2); - taosMemoryFreeClear(buffer); - return totalMsgs; -} - -void printParaIntoFile() { - // FILE *fp = fopen(g_stConfInfo.resultFileName, "a"); - TdFilePtr pFile = - taosOpenFile(g_stConfInfo.resultFileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_STREAM); - if (NULL == pFile) { - fprintf(stderr, "Failed to open %s for save result\n", g_stConfInfo.resultFileName); - exit(-1); - } - g_fp = pFile; - - time_t tTime = taosGetTimestampSec(); - struct tm tm; - taosLocalTime(&tTime, &tm); - - taosFprintfFile(pFile, "###################################################################\n"); - taosFprintfFile(pFile, "# configDir: %s\n", configDir); - taosFprintfFile(pFile, "# dbName: %s\n", g_stConfInfo.dbName); - taosFprintfFile(pFile, "# stbName: %s\n", g_stConfInfo.stbName); - taosFprintfFile(pFile, "# vnodeWalPath: %s\n", g_stConfInfo.vnodeWalPath); - taosFprintfFile(pFile, "# numOfTables: %d\n", g_stConfInfo.numOfTables); - taosFprintfFile(pFile, "# numOfThreads: %d\n", g_stConfInfo.numOfThreads); - taosFprintfFile(pFile, "# numOfVgroups: %d\n", g_stConfInfo.numOfVgroups); - taosFprintfFile(pFile, "# runMode: %d\n", g_stConfInfo.runMode); - taosFprintfFile(pFile, "# ratio: %f\n", g_stConfInfo.ratio); - taosFprintfFile(pFile, "# numOfColumn: %d\n", g_stConfInfo.numOfColumn); - taosFprintfFile(pFile, "# batchNumOfRow: %d\n", g_stConfInfo.batchNumOfRow); - taosFprintfFile(pFile, "# totalRowsOfPerTbl: %d\n", g_stConfInfo.totalRowsOfPerTbl); - taosFprintfFile(pFile, "# totalRowsOfT2: %d\n", g_stConfInfo.totalRowsOfT2); - taosFprintfFile(pFile, "# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - taosFprintfFile(pFile, "###################################################################\n"); - taosFprintfFile(pFile, - "|-------------------------------insert " - "info-----------------------------|--------------------------------consume " - "info---------------------------------|\n"); - taosFprintfFile(pFile, - "|batch size| insert msgs | insert time(s) | msgs/s | walLogSize(MB) | consume msgs | consume " - "time(s) | msgs/s | MB/s | avg msg size(KB) |\n"); - taosFprintfFile(g_fp, "|%10d", g_stConfInfo.batchNumOfRow); -} - -int main(int32_t argc, char* argv[]) { - parseArgument(argc, argv); - printParaIntoFile(); - - int64_t walLogSize = 0; - - int code; - code = init_env(); - if (code != 0) { - fprintf(stderr, "%% init_env error!\n"); - return -1; - } - - int32_t totalMsgs = 0; - - if (g_stConfInfo.runMode != TMQ_RUN_ONLY_CONSUME) { - int64_t startTs = taosGetTimestampUs(); - if (1 == g_stConfInfo.ratio) { - totalMsgs = syncWriteData(); - } else { - totalMsgs = syncWriteDataByRatio(); - } - - if (totalMsgs <= 0) { - pError("inset data error!\n"); - return -1; - } - int64_t endTs = taosGetTimestampUs(); - int64_t delay = endTs - startTs; - - int32_t totalRows = 0; - if (1 == g_stConfInfo.ratio) { - totalRows = g_stConfInfo.totalRowsOfPerTbl * g_stConfInfo.numOfTables; - } else { - totalRows = g_stConfInfo.totalRowsOfPerTbl * (1 + g_stConfInfo.ratio); - } - - float seconds = delay / 1000000.0; - float rowsSpeed = totalRows / seconds; - float msgsSpeed = totalMsgs / seconds; - - if ((0 == g_stConfInfo.simCase) && (strlen(g_stConfInfo.vnodeWalPath))) { - walLogSize = getDirectorySize(g_stConfInfo.vnodeWalPath); - if (walLogSize <= 0) { - printf("%s size incorrect!", g_stConfInfo.vnodeWalPath); - exit(-1); - } else { - pPrint(".log file size in vnode2/wal: %.3f MBytes\n", (double)walLogSize / (1024 * 1024.0)); - } - } - - if (0 == g_stConfInfo.simCase) { - pPrint("insert result: %d rows, %d msgs, time:%.3f sec, speed:%.1f rows/second, %.1f msgs/second\n", totalRows, - totalMsgs, seconds, rowsSpeed, msgsSpeed); - } - taosFprintfFile(g_fp, "|%10d | %10.3f | %8.2f | %10.3f ", totalMsgs, seconds, msgsSpeed, - (double)walLogSize / (1024 * 1024.0)); - } - - if (g_stConfInfo.runMode == TMQ_RUN_ONLY_INSERT) { - return 0; - } - - tmq_t* tmq = build_consumer(); - tmq_list_t* topic_list = build_topic_list(); - if ((NULL == tmq) || (NULL == topic_list)) { - return -1; - } - - perf_loop(tmq, topic_list, totalMsgs, walLogSize); - - taosMemoryFreeClear(g_pRowValue); - taosFprintfFile(g_fp, "\n"); - taosCloseFile(&g_fp); - return 0; -} - diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c deleted file mode 100644 index 71b31ba107..0000000000 --- a/tests/test/c/tmqSim.c +++ /dev/null @@ -1,1520 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "taos.h" -#include "taosdef.h" -#include "taoserror.h" -#include "tlog.h" -#include "types.h" - -#define GREEN "\033[1;32m" -#define NC "\033[0m" -#define min(a, b) (((a) < (b)) ? (a) : (b)) - -#define MAX_SQL_STR_LEN (1024 * 1024) -#define MAX_ROW_STR_LEN (16 * 1024) -#define MAX_CONSUMER_THREAD_CNT (16) -#define MAX_VGROUP_CNT (32) -#define SEND_TIME_UNIT 10 // ms -#define MAX_SQL_LEN 1048576 - -typedef enum { - NOTIFY_CMD_START_CONSUM, - NOTIFY_CMD_START_COMMIT, - NOTIFY_CMD_ID_BUTT, -} NOTIFY_CMD_ID; - -typedef enum enumQUERY_TYPE { - NO_INSERT_TYPE, - INSERT_TYPE, - QUERY_TYPE_BUT -} QUERY_TYPE; - -typedef struct { - TdThread thread; - int32_t consumerId; - - int32_t ifManualCommit; - // int32_t autoCommitIntervalMs; // 1000 ms - // char autoCommit[8]; // true, false - // char autoOffsetRest[16]; // none, earliest, latest - - TdFilePtr pConsumeRowsFile; - TdFilePtr pConsumeMetaFile; - int32_t ifCheckData; - int64_t expectMsgCnt; - - int64_t consumeMsgCnt; - int64_t consumeRowCnt; - int64_t consumeLen; - int32_t checkresult; - - char topicString[1024]; - char keyString[1024]; - - int32_t numOfTopic; - char topics[32][64]; - - int32_t numOfKey; - char key[32][64]; - char value[32][64]; - - tmq_t* tmq; - tmq_list_t* topicList; - - int32_t numOfVgroups; - int32_t rowsOfPerVgroups[MAX_VGROUP_CNT][2]; // [i][0]: vgroup id, [i][1]: rows of consume - int64_t ts; - - TAOS* taos; - - // below parameters is used by omb test - int32_t producerRate; // unit: msgs/s - int64_t totalProduceMsgs; - int64_t totalMsgsLen; - -} SThreadInfo; - -typedef struct { - // input from argvs - char cdbName[32]; - char dbName[64]; - int32_t showMsgFlag; - int32_t showRowFlag; - int32_t saveRowFlag; - int32_t consumeDelay; // unit s - int32_t numOfThread; - int32_t useSnapshot; - int64_t nowTime; - SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; - - SThreadInfo stProdThreads[MAX_CONSUMER_THREAD_CNT]; - - // below parameters is used by omb test - char topic[64]; - int32_t producers; - int32_t producerRate; - int32_t runDurationMinutes; - int32_t batchSize; - int32_t payloadLen; -} SConfInfo; - -static SConfInfo g_stConfInfo; -TdFilePtr g_fp = NULL; -static int running = 1; -char* g_payload = NULL; - -// char* g_pRowValue = NULL; -// TdFilePtr g_fp = NULL; - -static void printHelp() { - char indent[10] = " "; - printf("Used to test the tmq feature with sim cases\n"); - - printf("%s%s\n", indent, "-c"); - printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); - printf("%s%s\n", indent, "-d"); - printf("%s%s%s\n", indent, indent, "The name of the database for cosumer, no default "); - printf("%s%s\n", indent, "-g"); - printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag); - printf("%s%s\n", indent, "-r"); - printf("%s%s%s%d\n", indent, indent, "showRowFlag, default is ", g_stConfInfo.showRowFlag); - printf("%s%s\n", indent, "-s"); - printf("%s%s%s%d\n", indent, indent, "saveRowFlag, default is ", g_stConfInfo.saveRowFlag); - printf("%s%s\n", indent, "-y"); - printf("%s%s%s%ds\n", indent, indent, "consume delay, default is ", g_stConfInfo.consumeDelay); - printf("%s%s\n", indent, "-e"); - printf("%s%s%s%d\n", indent, indent, "snapshot, default is ", g_stConfInfo.useSnapshot); - - printf("%s%s\n", indent, "-t"); - printf("%s%s%s\n", indent, indent, "topic name, default is null"); - - printf("%s%s\n", indent, "-x"); - printf("%s%s%s\n", indent, indent, "consume thread number, default is 1"); - - - printf("%s%s\n", indent, "-l"); - printf("%s%s%s\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes); - printf("%s%s\n", indent, "-p"); - printf("%s%s%s\n", indent, indent, "producer thread number, default is 0"); - printf("%s%s\n", indent, "-b"); - printf("%s%s%s\n", indent, indent, "batch size, default is 1"); - printf("%s%s\n", indent, "-i"); - printf("%s%s%s\n", indent, indent, "produce rate unit is msgs /s, default is 100000"); - printf("%s%s\n", indent, "-n"); - printf("%s%s%s\n", indent, indent, "payload len unit is byte, default is 1000"); - - - exit(EXIT_SUCCESS); -} - -char* getCurrentTimeString(char* timeString) { - time_t tTime = taosGetTimestampSec(); - struct tm tm; - taosLocalTime(&tTime, &tm); - sprintf(timeString, "%d-%02d-%02d %02d:%02d:%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, - tm.tm_min, tm.tm_sec); - - return timeString; -} - -static void tmqStop(int signum, void* info, void* ctx) { - running = 0; - char tmpString[128]; - taosFprintfFile(g_fp, "%s tmqStop() receive stop signal[%d]\n", getCurrentTimeString(tmpString), signum); -} - -static void tmqSetSignalHandle() { taosSetSignal(SIGINT, tmqStop); } - -void initLogFile() { - char filename[256]; - char tmpString[128]; - - pid_t process_id = getpid(); - - if (0 != strlen(g_stConfInfo.topic)) { - sprintf(filename, "/tmp/tmqlog-%d-%s.txt", process_id, getCurrentTimeString(tmpString)); - } else { - sprintf(filename, "%s/../log/tmqlog-%d-%s.txt", configDir, process_id, getCurrentTimeString(tmpString)); - } -#ifdef WINDOWS - for (int i = 2; i < sizeof(filename); i++) { - if (filename[i] == ':') filename[i] = '-'; - if (filename[i] == '\0') break; - } -#endif - TdFilePtr pFile = taosOpenFile(filename, TD_FILE_TEXT | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); - if (NULL == pFile) { - fprintf(stderr, "Failed to open %s for save result\n", filename); - exit(-1); - } - g_fp = pFile; -} - -void saveConfigToLogFile() { - taosFprintfFile(g_fp, "###################################################################\n"); - taosFprintfFile(g_fp, "# configDir: %s\n", configDir); - taosFprintfFile(g_fp, "# dbName: %s\n", g_stConfInfo.dbName); - taosFprintfFile(g_fp, "# cdbName: %s\n", g_stConfInfo.cdbName); - taosFprintfFile(g_fp, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); - taosFprintfFile(g_fp, "# showRowFlag: %d\n", g_stConfInfo.showRowFlag); - taosFprintfFile(g_fp, "# saveRowFlag: %d\n", g_stConfInfo.saveRowFlag); - taosFprintfFile(g_fp, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); - taosFprintfFile(g_fp, "# numOfThread: %d\n", g_stConfInfo.numOfThread); - - for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { - taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); - // taosFprintfFile(g_fp, " auto commit: %s\n", g_stConfInfo.stThreads[i].autoCommit); - // taosFprintfFile(g_fp, " auto commit interval ms: %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs); - // taosFprintfFile(g_fp, " auto offset rest: %s\n", g_stConfInfo.stThreads[i].autoOffsetRest); - taosFprintfFile(g_fp, " Topics: "); - for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) { - taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]); - } - taosFprintfFile(g_fp, "\n"); - taosFprintfFile(g_fp, " Key: "); - for (int k = 0; k < g_stConfInfo.stThreads[i].numOfKey; k++) { - taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]); - } - taosFprintfFile(g_fp, "\n"); - taosFprintfFile(g_fp, " expect rows: %d\n", g_stConfInfo.stThreads[i].expectMsgCnt); - } - - char tmpString[128]; - taosFprintfFile(g_fp, "# Test time: %s\n", getCurrentTimeString(tmpString)); - taosFprintfFile(g_fp, "###################################################################\n"); -} - -void parseArgument(int32_t argc, char* argv[]) { - memset(&g_stConfInfo, 0, sizeof(SConfInfo)); - g_stConfInfo.showMsgFlag = 0; - g_stConfInfo.showRowFlag = 0; - g_stConfInfo.saveRowFlag = 0; - g_stConfInfo.consumeDelay = 5; - g_stConfInfo.numOfThread = 1; - g_stConfInfo.batchSize = 1; - g_stConfInfo.producers = 0; - - g_stConfInfo.nowTime = taosGetTimestampMs(); - - for (int32_t i = 1; i < argc; i++) { - if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { - printHelp(); - exit(0); - } else if (strcmp(argv[i], "-d") == 0) { - strcpy(g_stConfInfo.dbName, argv[++i]); - } else if (strcmp(argv[i], "-w") == 0) { - strcpy(g_stConfInfo.cdbName, argv[++i]); - } else if (strcmp(argv[i], "-c") == 0) { - strcpy(configDir, argv[++i]); - } else if (strcmp(argv[i], "-g") == 0) { - g_stConfInfo.showMsgFlag = atol(argv[++i]); - } else if (strcmp(argv[i], "-r") == 0) { - g_stConfInfo.showRowFlag = atol(argv[++i]); - } else if (strcmp(argv[i], "-s") == 0) { - g_stConfInfo.saveRowFlag = atol(argv[++i]); - } else if (strcmp(argv[i], "-y") == 0) { - g_stConfInfo.consumeDelay = atol(argv[++i]); - } else if (strcmp(argv[i], "-e") == 0) { - g_stConfInfo.useSnapshot = atol(argv[++i]); - } else if (strcmp(argv[i], "-t") == 0) { - char tmpBuf[56]; - strcpy(tmpBuf, argv[++i]); - sprintf(g_stConfInfo.topic, "`%s`", tmpBuf); - } else if (strcmp(argv[i], "-x") == 0) { - g_stConfInfo.numOfThread = atol(argv[++i]); - } else if (strcmp(argv[i], "-l") == 0) { - g_stConfInfo.runDurationMinutes = atol(argv[++i]); - } else if (strcmp(argv[i], "-p") == 0) { - g_stConfInfo.producers = atol(argv[++i]); - } else if (strcmp(argv[i], "-b") == 0) { - g_stConfInfo.batchSize = atol(argv[++i]); - } else if (strcmp(argv[i], "-i") == 0) { - g_stConfInfo.producerRate = atol(argv[++i]); - } else if (strcmp(argv[i], "-n") == 0) { - g_stConfInfo.payloadLen = atol(argv[++i]); - } else { - pError("%s unknow para: %s %s", GREEN, argv[++i], NC); - exit(-1); - } - } - - g_payload = taosMemoryCalloc(g_stConfInfo.payloadLen + 1, 1); - if (NULL == g_payload) { - pPrint("%s failed to malloc for payload %s", GREEN, NC); - exit(-1); - } - - for (int32_t i = 0; i < g_stConfInfo.payloadLen; i++) { - strcpy(&g_payload[i], "a"); - } - - initLogFile(); - - taosFprintfFile(g_fp, "====parseArgument() success\n"); - -#if 1 - pPrint("%s configDir:%s %s", GREEN, configDir, NC); - pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC); - pPrint("%s cdbName:%s %s", GREEN, g_stConfInfo.cdbName, NC); - pPrint("%s consumeDelay:%d %s", GREEN, g_stConfInfo.consumeDelay, NC); - pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); - pPrint("%s showRowFlag:%d %s", GREEN, g_stConfInfo.showRowFlag, NC); - pPrint("%s saveRowFlag:%d %s", GREEN, g_stConfInfo.saveRowFlag, NC); - - pPrint("%s snapshot:%d %s", GREEN, g_stConfInfo.useSnapshot, NC); - - pPrint("%s omb topic:%s %s", GREEN, g_stConfInfo.topic, NC); - pPrint("%s numOfThread:%d %s", GREEN, g_stConfInfo.numOfThread, NC); -#endif -} - -void splitStr(char** arr, char* str, const char* del) { - char* s = strtok(str, del); - while (s != NULL) { - *arr++ = s; - s = strtok(NULL, del); - } -} - -void ltrim(char* str) { - if (str == NULL || *str == '\0') { - return; - } - int len = 0; - char* p = str; - while (*p != '\0' && isspace(*p)) { - ++p; - ++len; - } - memmove(str, p, strlen(str) - len + 1); - // return str; -} - -int queryDB(TAOS* taos, char* command) { - int retryCnt = 10; - int code; - TAOS_RES* pRes; - - while (retryCnt--) { - pRes = taos_query(taos, command); - code = taos_errno(pRes); - if (code != 0) { - taosSsleep(1); - taos_free_result(pRes); - continue; - } - taos_free_result(pRes); - return 0; - } - - pError("failed to reason:%s, sql: %s", tstrerror(code), command); - taos_free_result(pRes); - return -1; -} - - -void addRowsToVgroupId(SThreadInfo* pInfo, int32_t vgroupId, int32_t rows) { - int32_t i; - for (i = 0; i < pInfo->numOfVgroups; i++) { - if (vgroupId == pInfo->rowsOfPerVgroups[i][0]) { - pInfo->rowsOfPerVgroups[i][1] += rows; - return; - } - } - - pInfo->rowsOfPerVgroups[pInfo->numOfVgroups][0] = vgroupId; - pInfo->rowsOfPerVgroups[pInfo->numOfVgroups][1] += rows; - pInfo->numOfVgroups++; - - taosFprintfFile(g_fp, "consume id %d, add one new vogroup id: %d\n", pInfo->consumerId, vgroupId); - if (pInfo->numOfVgroups > MAX_VGROUP_CNT) { - taosFprintfFile(g_fp, "====consume id %d, vgroup num %d over than 32. new vgroupId: %d\n", pInfo->consumerId, - pInfo->numOfVgroups, vgroupId); - taosCloseFile(&g_fp); - exit(-1); - } -} - -TAOS* createNewTaosConnect() { - TAOS* taos = NULL; - int32_t retryCnt = 10; - - while (retryCnt--) { - TAOS* taos = taos_connect(NULL, "root", "taosdata", NULL, 0); - if (NULL != taos) { - return taos; - } - taosSsleep(1); - } - - taosFprintfFile(g_fp, "taos_connect() fail\n"); - return NULL; -} - - -int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) { - char sqlStr[1100] = {0}; - - if (strlen(buf) > 1024) { - taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", strlen(buf)); - taosCloseFile(&g_fp); - return -1; - } - - TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); - if (pConn == NULL) { - taosFprintfFile(g_fp, "taos_connect() fail, can not save consume result to main script\n"); - return -1; - } - - sprintf(sqlStr, "insert into %s.content_%d values (%" PRId64 ", \'%s\')", g_stConfInfo.cdbName, pInfo->consumerId, - pInfo->ts++, buf); - int retCode = queryDB(pConn, sqlStr); - if (retCode != 0) { - taosFprintfFile(g_fp, "error in save consume content\n"); - taosCloseFile(&g_fp); - taos_close(pConn); - exit(-1); - } - - taos_close(pConn); - - return 0; -} - -static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) { - // if (shell.args.is_raw_time) { - // sprintf(buf, "%" PRId64, val); - // return buf; - // } - - time_t tt; - int32_t ms = 0; - if (precision == TSDB_TIME_PRECISION_NANO) { - tt = (time_t)(val / 1000000000); - ms = val % 1000000000; - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - tt = (time_t)(val / 1000000); - ms = val % 1000000; - } else { - tt = (time_t)(val / 1000); - ms = val % 1000; - } - - if (tt <= 0 && ms < 0) { - tt--; - if (precision == TSDB_TIME_PRECISION_NANO) { - ms += 1000000000; - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - ms += 1000000; - } else { - ms += 1000; - } - } - - struct tm ptm; - taosLocalTime(&tt, &ptm); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); - - if (precision == TSDB_TIME_PRECISION_NANO) { - sprintf(buf + pos, ".%09d", ms); - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - sprintf(buf + pos, ".%06d", ms); - } else { - sprintf(buf + pos, ".%03d", ms); - } - - return buf; -} - -static void shellDumpFieldToFile(TdFilePtr pFile, const char* val, TAOS_FIELD* field, int32_t length, - int32_t precision) { - if (val == NULL) { - return; - } - - int n; - char buf[TSDB_MAX_BYTES_PER_ROW]; - switch (field->type) { - case TSDB_DATA_TYPE_BOOL: - taosFprintfFile(pFile, "%d", ((((int32_t)(*((char*)val))) == 1) ? 1 : 0)); - break; - case TSDB_DATA_TYPE_TINYINT: - taosFprintfFile(pFile, "%d", *((int8_t*)val)); - break; - case TSDB_DATA_TYPE_UTINYINT: - taosFprintfFile(pFile, "%u", *((uint8_t*)val)); - break; - case TSDB_DATA_TYPE_SMALLINT: - taosFprintfFile(pFile, "%d", *((int16_t*)val)); - break; - case TSDB_DATA_TYPE_USMALLINT: - taosFprintfFile(pFile, "%u", *((uint16_t*)val)); - break; - case TSDB_DATA_TYPE_INT: - taosFprintfFile(pFile, "%d", *((int32_t*)val)); - break; - case TSDB_DATA_TYPE_UINT: - taosFprintfFile(pFile, "%u", *((uint32_t*)val)); - break; - case TSDB_DATA_TYPE_BIGINT: - taosFprintfFile(pFile, "%" PRId64, *((int64_t*)val)); - break; - case TSDB_DATA_TYPE_UBIGINT: - taosFprintfFile(pFile, "%" PRIu64, *((uint64_t*)val)); - break; - case TSDB_DATA_TYPE_FLOAT: - taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val)); - break; - case TSDB_DATA_TYPE_DOUBLE: - n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val)); - if (n > TMAX(25, length)) { - taosFprintfFile(pFile, "%*.15e", length, GET_DOUBLE_VAL(val)); - } else { - taosFprintfFile(pFile, "%s", buf); - } - break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_JSON: - { - char quotationStr[2]; - int32_t bufIndex = 0; - quotationStr[0] = 0; - quotationStr[1] = 0; - for (int32_t i = 0; i < length; i++) { - buf[bufIndex] = val[i]; - bufIndex++; - if (val[i] == '\"') { - buf[bufIndex] = val[i]; - bufIndex++; - quotationStr[0] = '\"'; - } - if (val[i] == ',') { - quotationStr[0] = '\"'; - } - } - buf[bufIndex] = 0; - if (length == 0) { - quotationStr[0] = '\"'; - } - - taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr); - } - break; - case TSDB_DATA_TYPE_TIMESTAMP: - shellFormatTimestamp(buf, *(int64_t*)val, precision); - taosFprintfFile(pFile, "%s", buf); - break; - default: - break; - } -} - -static void dumpToFileForCheck(TdFilePtr pFile, TAOS_ROW row, TAOS_FIELD* fields, int32_t* length, int32_t num_fields, - int32_t precision) { - for (int32_t i = 0; i < num_fields; i++) { - if (i > 0) { - taosFprintfFile(pFile, ","); - } - shellDumpFieldToFile(pFile, (const char*)row[i], fields + i, length[i], precision); - } - taosFprintfFile(pFile, "\n"); -} - -static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) { - char buf[1024]; - int32_t totalRows = 0; - - // printf("topic: %s\n", tmq_get_topic_name(msg)); - int32_t vgroupId = tmq_get_vgroup_id(msg); - const char* dbName = tmq_get_db_name(msg); - - taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); - taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", - tmq_get_topic_name(msg), vgroupId); - - while (1) { - TAOS_ROW row = taos_fetch_row(msg); - - if (row == NULL) break; - - TAOS_FIELD* fields = taos_fetch_fields(msg); - int32_t numOfFields = taos_field_count(msg); - int32_t* length = taos_fetch_lengths(msg); - int32_t precision = taos_result_precision(msg); - const char* tbName = tmq_get_table_name(msg); - -#if 0 - // get schema - //============================== stub =================================================// - for (int32_t i = 0; i < numOfFields; i++) { - taosFprintfFile(g_fp, "%02d: name: %s, type: %d, len: %d\n", i, fields[i].name, fields[i].type, fields[i].bytes); - } - //============================== stub =================================================// -#endif - - dumpToFileForCheck(pInfo->pConsumeRowsFile, row, fields, length, numOfFields, precision); - - taos_print_row(buf, row, fields, numOfFields); - - if (0 != g_stConfInfo.showRowFlag) { - taosFprintfFile(g_fp, "tbname:%s, rows[%d]: %s\n", (tbName != NULL ? tbName : "null table"), totalRows, buf); - // if (0 != g_stConfInfo.saveRowFlag) { - // saveConsumeContentToTbl(pInfo, buf); - // } - } - - totalRows++; - } - - addRowsToVgroupId(pInfo, vgroupId, totalRows); - - return totalRows; -} - - -static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) { - char buf[1024]; - int32_t totalRows = 0; - - // printf("topic: %s\n", tmq_get_topic_name(msg)); - int32_t vgroupId = tmq_get_vgroup_id(msg); - const char* dbName = tmq_get_db_name(msg); - - taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); - taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", - tmq_get_topic_name(msg), vgroupId); - - { - tmq_raw_data raw = {0}; - int32_t code = tmq_get_raw(msg, &raw); - - if(code == TSDB_CODE_SUCCESS){ - int retCode = queryDB(pInfo->taos, "use metadb"); - if (retCode != 0) { - taosFprintfFile(g_fp, "error when use metadb\n"); - taosCloseFile(&g_fp); - exit(-1); - } - taosFprintfFile(g_fp, "raw:%p\n", &raw); - - tmq_write_raw(pInfo->taos, raw); - } - - char* result = tmq_get_json_meta(msg); - if(result){ - //printf("meta result: %s\n", result); - taosFprintfFile(pInfo->pConsumeMetaFile, "%s\n", result); - taosMemoryFree(result); - } - } - - totalRows++; - - return totalRows; -} - -static void appNothing(void* param, TAOS_RES* res, int32_t numOfRows) {} - -int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) { - char sqlStr[1024] = {0}; - - // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int - sprintf(sqlStr, "insert into %s.notifyinfo values (%" PRId64 ", %d, %d)", g_stConfInfo.cdbName, atomic_fetch_add_64(&g_stConfInfo.nowTime, 1), cmdId, - pInfo->consumerId); - - taos_query_a(pInfo->taos, sqlStr, appNothing, NULL); - - taosFprintfFile(g_fp, "notifyMainScript success, sql: %s\n", sqlStr); - - return 0; -} - -static int32_t g_once_commit_flag = 0; -static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { - taosFprintfFile(g_fp, "tmq_commit_cb_print() commit %d\n", code); - - if (0 == g_once_commit_flag) { - g_once_commit_flag = 1; - notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT); - } - - char tmpString[128]; - taosFprintfFile(g_fp, "%s tmq_commit_cb_print() be called\n", getCurrentTimeString(tmpString)); -} - -void build_consumer(SThreadInfo* pInfo) { - tmq_conf_t* conf = tmq_conf_new(); - - // tmq_conf_set(conf, "td.connect.ip", "localhost"); - // tmq_conf_set(conf, "td.connect.port", "6030"); - tmq_conf_set(conf, "td.connect.user", "root"); - tmq_conf_set(conf, "td.connect.pass", "taosdata"); - - // tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); - - tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, pInfo); - - // tmq_conf_set(conf, "group.id", "cgrp1"); - for (int32_t i = 0; i < pInfo->numOfKey; i++) { - tmq_conf_set(conf, pInfo->key[i], pInfo->value[i]); - } - - tmq_conf_set(conf, "msg.with.table.name", "true"); - - // tmq_conf_set(conf, "client.id", "c-001"); - - // tmq_conf_set(conf, "enable.auto.commit", "true"); - // tmq_conf_set(conf, "enable.auto.commit", "false"); - - // tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); - - // tmq_conf_set(conf, "auto.offset.reset", "none"); - // tmq_conf_set(conf, "auto.offset.reset", "earliest"); - // tmq_conf_set(conf, "auto.offset.reset", "latest"); - // - if (g_stConfInfo.useSnapshot) { - tmq_conf_set(conf, "experimental.snapshot.enable", "true"); - } - - pInfo->tmq = tmq_consumer_new(conf, NULL, 0); - - tmq_conf_destroy(conf); - - return; -} - -void build_topic_list(SThreadInfo* pInfo) { - pInfo->topicList = tmq_list_new(); - // tmq_list_append(topic_list, "test_stb_topic_1"); - for (int32_t i = 0; i < pInfo->numOfTopic; i++) { - tmq_list_append(pInfo->topicList, pInfo->topics[i]); - } - return; -} - -int32_t saveConsumeResult(SThreadInfo* pInfo) { - char sqlStr[1024] = {0}; - // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int - sprintf(sqlStr, "insert into %s.consumeresult values (%" PRId64 ", %d, %" PRId64 ", %" PRId64 ", %d)", - g_stConfInfo.cdbName, atomic_fetch_add_64(&g_stConfInfo.nowTime, 1), pInfo->consumerId, pInfo->consumeMsgCnt, - pInfo->consumeRowCnt, pInfo->checkresult); - - char tmpString[128]; - taosFprintfFile(g_fp, "%s, consume id %d result: %s\n", getCurrentTimeString(tmpString), pInfo->consumerId, sqlStr); - - int retCode = queryDB(pInfo->taos, sqlStr); - if (retCode != 0) { - taosFprintfFile(g_fp, "consume id %d error in save consume result\n", pInfo->consumerId); - return -1; - } - - return 0; -} - -void loop_consume(SThreadInfo* pInfo) { - int32_t code; - - int32_t once_flag = 0; - - int64_t totalMsgs = 0; - int64_t totalRows = 0; - - char tmpString[128]; - taosFprintfFile(g_fp, "%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), - pInfo->consumerId); - - pInfo->ts = taosGetTimestampMs(); - - if (pInfo->ifCheckData) { - char filename[256] = {0}; - char tmpString[128]; - // sprintf(filename, "%s/../log/consumerid_%d_%s.txt", configDir, pInfo->consumerId, - // getCurrentTimeString(tmpString)); - sprintf(filename, "%s/../log/consumerid_%d.txt", configDir, pInfo->consumerId); - pInfo->pConsumeRowsFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); - - sprintf(filename, "%s/../log/meta_consumerid_%d.txt", configDir, pInfo->consumerId); - pInfo->pConsumeMetaFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); - - if (pInfo->pConsumeRowsFile == NULL || pInfo->pConsumeMetaFile == NULL) { - taosFprintfFile(g_fp, "%s create file fail for save rows or save meta\n", getCurrentTimeString(tmpString)); - return; - } - } - - int64_t lastTotalMsgs = 0; - uint64_t lastPrintTime = taosGetTimestampMs(); - uint64_t startTs = taosGetTimestampMs(); - - int32_t consumeDelay = g_stConfInfo.consumeDelay == -1 ? -1 : (g_stConfInfo.consumeDelay * 1000); - while (running) { - TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, consumeDelay); - if (tmqMsg) { - if (0 != g_stConfInfo.showMsgFlag) { - tmq_res_t msgType = tmq_get_res_type(tmqMsg); - if (msgType == TMQ_RES_TABLE_META) { - totalRows += meta_msg_process(tmqMsg, pInfo, totalMsgs); - } else if (msgType == TMQ_RES_DATA) - totalRows += data_msg_process(tmqMsg, pInfo, totalMsgs); - } - - taos_free_result(tmqMsg); - - totalMsgs++; - - int64_t currentPrintTime = taosGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 10 * 1000) { - taosFprintfFile( - g_fp, "consumer id %d has currently poll total msgs: %" PRId64 ", period rate: %.3f msgs/second\n", - pInfo->consumerId, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0 / (currentPrintTime - lastPrintTime)); - lastPrintTime = currentPrintTime; - lastTotalMsgs = totalMsgs; - } - - if (0 == once_flag) { - once_flag = 1; - notifyMainScript(pInfo, NOTIFY_CMD_START_CONSUM); - } - - if ((totalRows >= pInfo->expectMsgCnt) || (totalMsgs >= pInfo->expectMsgCnt)) { - char tmpString[128]; - taosFprintfFile(g_fp, "%s over than expect rows, so break consume\n", getCurrentTimeString(tmpString)); - break; - } - } else { - char tmpString[128]; - taosFprintfFile(g_fp, "%s no poll more msg when time over, break consume\n", getCurrentTimeString(tmpString)); - break; - } - } - - if (0 == running) { - taosFprintfFile(g_fp, "receive stop signal and not continue consume\n"); - } - - pInfo->consumeMsgCnt = totalMsgs; - pInfo->consumeRowCnt = totalRows; - - taosFprintfFile(g_fp, "==== consumerId: %d, consumeMsgCnt: %" PRId64 ", consumeRowCnt: %" PRId64 "\n", - pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt); -} - -void* consumeThreadFunc(void* param) { - SThreadInfo* pInfo = (SThreadInfo*)param; - - pInfo->taos = createNewTaosConnect(); - if (pInfo->taos == NULL) { - taosFprintfFile(g_fp, "taos_connect() fail, can not notify and save consume result to main scripte\n"); - return NULL; - } - - build_consumer(pInfo); - build_topic_list(pInfo); - if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { - taosFprintfFile(g_fp, "create consumer fail! tmq is null or topicList is null\n"); - taos_close(pInfo->taos); - pInfo->taos = NULL; - return NULL; - } - - int32_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); - if (err != 0) { - pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); - taosFprintfFile(g_fp, "tmq_subscribe() fail! reason: %s\n", tmq_err2str(err)); - taos_close(pInfo->taos); - pInfo->taos = NULL; - return NULL; - } - - tmq_list_destroy(pInfo->topicList); - pInfo->topicList = NULL; - - loop_consume(pInfo); - - if (pInfo->ifManualCommit) { - pPrint("tmq_commit() manual commit when consume end.\n"); - /*tmq_commit(pInfo->tmq, NULL, 0);*/ - tmq_commit_sync(pInfo->tmq, NULL); - taosFprintfFile(g_fp, "tmq_commit() manual commit over.\n"); - pPrint("tmq_commit() manual commit over.\n"); - } - - err = tmq_unsubscribe(pInfo->tmq); - if (err != 0) { - pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); - taosFprintfFile(g_fp, "tmq_unsubscribe()! reason: %s\n", tmq_err2str(err)); - } - - err = tmq_consumer_close(pInfo->tmq); - if (err != 0) { - pError("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); - taosFprintfFile(g_fp, "tmq_consumer_close()! reason: %s\n", tmq_err2str(err)); - } - pInfo->tmq = NULL; - - // save consume result into consumeresult table - saveConsumeResult(pInfo); - - // save rows from per vgroup - taosFprintfFile(g_fp, "======== consumerId: %d, consume rows from per vgroups ========\n", pInfo->consumerId); - for (int32_t i = 0; i < pInfo->numOfVgroups; i++) { - taosFprintfFile(g_fp, "vgroups: %04d, rows: %d\n", pInfo->rowsOfPerVgroups[i][0], pInfo->rowsOfPerVgroups[i][1]); - } - - taos_close(pInfo->taos); - pInfo->taos = NULL; - - return NULL; -} - -void parseConsumeInfo() { - char* token; - const char delim[2] = ","; - const char ch = ':'; - - for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { - token = strtok(g_stConfInfo.stThreads[i].topicString, delim); - while (token != NULL) { - // printf("%s\n", token ); - strcpy(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic], token); - ltrim(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic]); - // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); - g_stConfInfo.stThreads[i].numOfTopic++; - - token = strtok(NULL, delim); - } - - token = strtok(g_stConfInfo.stThreads[i].keyString, delim); - while (token != NULL) { - // printf("%s\n", token ); - { - char* pstr = token; - ltrim(pstr); - char* ret = strchr(pstr, ch); - memcpy(g_stConfInfo.stThreads[i].key[g_stConfInfo.stThreads[i].numOfKey], pstr, ret - pstr); - strcpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1); - // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], - // g_stConfInfo.value[g_stConfInfo.numOfKey]); - g_stConfInfo.stThreads[i].numOfKey++; - } - - token = strtok(NULL, delim); - } - } -} - -int32_t getConsumeInfo() { - char sqlStr[1024] = {0}; - - TAOS* pConn = createNewTaosConnect(); - if (pConn == NULL) { - taosFprintfFile(g_fp, "taos_connect() fail, can not get consume info for start consumer\n"); - return -1; - } - - sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.cdbName); - TAOS_RES *pRes = taos_query(pConn, sqlStr); - if (taos_errno(pRes) != 0) { - taosFprintfFile(g_fp, "error in get consumeinfo for %s\n", taos_errstr(pRes)); - taosCloseFile(&g_fp); - taos_free_result(pRes); - taos_close(pConn); - return -1; - } - - TAOS_ROW row = NULL; - int num_fields = taos_num_fields(pRes); - TAOS_FIELD* fields = taos_fetch_fields(pRes); - - // schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, - // ifcheckdata int - - int32_t numOfThread = 0; - while ((row = taos_fetch_row(pRes))) { - int32_t* lengths = taos_fetch_lengths(pRes); - - // set default value - // g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000; - // memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true")); - // memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast")); - - for (int i = 0; i < num_fields; ++i) { - if (row[i] == NULL || 0 == i) { - continue; - } - - if ((1 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { - g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t*)row[i]); - } else if ((2 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { - memcpy(g_stConfInfo.stThreads[numOfThread].topicString, row[i], lengths[i]); - } else if ((3 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { - memcpy(g_stConfInfo.stThreads[numOfThread].keyString, row[i], lengths[i]); - } else if ((4 == i) && (fields[i].type == TSDB_DATA_TYPE_BIGINT)) { - g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]); - } else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { - g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]); - } else if ((6 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { - g_stConfInfo.stThreads[numOfThread].ifManualCommit = *((int32_t*)row[i]); - } - } - numOfThread++; - } - g_stConfInfo.numOfThread = numOfThread; - - taos_free_result(pRes); - - parseConsumeInfo(); - taos_close(pConn); - - return 0; -} - - -static int32_t omb_data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex, int64_t* lenOfRows) { - char buf[16*1024]; - int32_t totalRows = 0; - int32_t totalLen = 0; - - // printf("topic: %s\n", tmq_get_topic_name(msg)); - //int32_t vgroupId = tmq_get_vgroup_id(msg); - //const char* dbName = tmq_get_db_name(msg); - - //taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); - //taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", - // tmq_get_topic_name(msg), vgroupId); - - while (1) { - TAOS_ROW row = taos_fetch_row(msg); - - if (row == NULL) break; - - TAOS_FIELD* fields = taos_fetch_fields(msg); - int32_t numOfFields = taos_field_count(msg); - //int32_t* length = taos_fetch_lengths(msg); - //int32_t precision = taos_result_precision(msg); - //const char* tbName = tmq_get_table_name(msg); - - taos_print_row(buf, row, fields, numOfFields); - totalLen += strlen(buf); - totalRows++; - } - - *lenOfRows = totalLen; - return totalRows; -} - -void omb_loop_consume(SThreadInfo* pInfo) { - int32_t code; - - int32_t once_flag = 0; - - int64_t totalMsgs = 0; - int64_t totalRows = 0; - - char tmpString[128]; - taosFprintfFile(g_fp, "%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), - pInfo->consumerId); - printf("%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), - pInfo->consumerId); - - pInfo->ts = taosGetTimestampMs(); - - int64_t lastTotalMsgs = 0; - uint64_t lastPrintTime = taosGetTimestampMs(); - uint64_t startTs = taosGetTimestampMs(); - - int64_t totalLenOfMsg = 0; - int64_t lastTotalLenOfMsg = 0; - int32_t consumeDelay = g_stConfInfo.consumeDelay == -1 ? -1 : (g_stConfInfo.consumeDelay * 1000); - while (running) { - TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, consumeDelay); - if (tmqMsg) { - int64_t lenOfMsg = 0; - totalRows += omb_data_msg_process(tmqMsg, pInfo, totalMsgs, &lenOfMsg); - totalLenOfMsg += lenOfMsg; - taos_free_result(tmqMsg); - totalMsgs++; - int64_t currentPrintTime = taosGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 10 * 1000) { - int64_t currentLenOfMsg = totalLenOfMsg - lastTotalLenOfMsg; - int64_t deltaTime = currentPrintTime - lastPrintTime; - printf("consumer id %d has currently cons total rows: %" PRId64 ", msgs: %" PRId64 ", rate: %.3f msgs/s, %.1f MB/s\n", - pInfo->consumerId, totalRows, totalMsgs, - (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime, - currentLenOfMsg*1000.0/(1024*1024)/deltaTime); - - taosFprintfFile( - g_fp, "consumer id %d has currently poll total msgs: %" PRId64 ", period cons rate: %.3f msgs/s, %.1f MB/s\n", - pInfo->consumerId, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime, currentLenOfMsg*1000.0/deltaTime); - lastPrintTime = currentPrintTime; - lastTotalMsgs = totalMsgs; - lastTotalLenOfMsg = totalLenOfMsg; - } - } else { - char tmpString[128]; - taosFprintfFile(g_fp, "%s no poll more msg when time over, break consume\n", getCurrentTimeString(tmpString)); - printf("%s no poll more msg when time over, break consume\n", getCurrentTimeString(tmpString)); - int64_t currentPrintTime = taosGetTimestampMs(); - int64_t currentLenOfMsg = totalLenOfMsg - lastTotalLenOfMsg; - int64_t deltaTime = currentPrintTime - lastPrintTime; - printf("consumer id %d has currently cons total rows: %" PRId64 ", msgs: %" PRId64 ", rate: %.3f msgs/s, %.1f MB/s\n", - pInfo->consumerId, totalRows, totalMsgs, - (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime, - currentLenOfMsg*1000.0/(1024*1024)/deltaTime); - break; - } - } - - pInfo->consumeMsgCnt = totalMsgs; - pInfo->consumeRowCnt = totalRows; - pInfo->consumeLen = totalLenOfMsg; - -} - - -void* ombConsumeThreadFunc(void* param) { - SThreadInfo* pInfo = (SThreadInfo*)param; - - //################### set key ######################## - tmq_conf_t* conf = tmq_conf_new(); - // tmq_conf_set(conf, "td.connect.ip", "localhost"); - // tmq_conf_set(conf, "td.connect.port", "6030"); - tmq_conf_set(conf, "td.connect.user", "root"); - tmq_conf_set(conf, "td.connect.pass", "taosdata"); - // tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); - tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, pInfo); - tmq_conf_set(conf, "group.id", "ombCgrp"); - // tmq_conf_set(conf, "msg.with.table.name", "true"); - // tmq_conf_set(conf, "client.id", "c-001"); - // tmq_conf_set(conf, "enable.auto.commit", "true"); - tmq_conf_set(conf, "enable.auto.commit", "false"); - // tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); - // tmq_conf_set(conf, "auto.offset.reset", "none"); - // tmq_conf_set(conf, "auto.offset.reset", "earliest"); - tmq_conf_set(conf, "auto.offset.reset", "earliest"); - // - if (g_stConfInfo.useSnapshot) { - tmq_conf_set(conf, "experimental.snapshot.enable", "true"); - } - - pInfo->tmq = tmq_consumer_new(conf, NULL, 0); - - tmq_conf_destroy(conf); - - //################### set topic ########################## - pInfo->topicList = tmq_list_new(); - tmq_list_append(pInfo->topicList, g_stConfInfo.topic); - - if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { - taosFprintfFile(g_fp, "create consumer fail! tmq is null or topicList is null\n"); - return NULL; - } - - int32_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); - if (err != 0) { - pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); - taosFprintfFile(g_fp, "tmq_subscribe() fail! reason: %s\n", tmq_err2str(err)); - return NULL; - } - - tmq_list_destroy(pInfo->topicList); - pInfo->topicList = NULL; - - omb_loop_consume(pInfo); - - err = tmq_unsubscribe(pInfo->tmq); - if (err != 0) { - pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); - taosFprintfFile(g_fp, "tmq_unsubscribe()! reason: %s\n", tmq_err2str(err)); - } - - err = tmq_consumer_close(pInfo->tmq); - if (err != 0) { - pError("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); - taosFprintfFile(g_fp, "tmq_consumer_close()! reason: %s\n", tmq_err2str(err)); - } - pInfo->tmq = NULL; - - return NULL; -} - - - -static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type) { - TAOS_RES *res = taos_query(taos, command); - int32_t code = taos_errno(res); - - if (code != 0) { - pPrint("%s Failed to execute <%s>, reason: %s %s", GREEN, command, taos_errstr(res), NC); - taos_free_result(res); - return -1; - } - - if (INSERT_TYPE == type) { - int affectedRows = taos_affected_rows(res); - taos_free_result(res); - return affectedRows; - } - - taos_free_result(res); - return 0; -} - -void* ombProduceThreadFunc(void* param) { - SThreadInfo* pInfo = (SThreadInfo*)param; - - pInfo->taos = createNewTaosConnect(); - if (pInfo->taos == NULL) { - taosFprintfFile(g_fp, "taos_connect() fail, can not start producers!\n"); - return NULL; - } - - int64_t affectedRowsTotal = 0; - int64_t sendMsgs = 0; - - uint32_t totalSendLoopTimes = g_stConfInfo.runDurationMinutes * 60 * 1000 / SEND_TIME_UNIT; // send some msgs per 10ms - uint32_t batchPerTblTimes = pInfo->producerRate / 100 / g_stConfInfo.batchSize; - uint32_t remainder = (pInfo->producerRate / 100) % g_stConfInfo.batchSize; - if (remainder) { - batchPerTblTimes += 1; - } - - char* sqlBuf = taosMemoryMalloc(MAX_SQL_LEN); - if (NULL == sqlBuf) { - printf("malloc fail for sqlBuf\n"); - taos_close(pInfo->taos); - pInfo->taos = NULL; - return NULL; - } - - printf("Produce Info: totalSendLoopTimes: %d, batchPerTblTimes: %d, producerRate: %d\n", totalSendLoopTimes, batchPerTblTimes, pInfo->producerRate); - - char ctbName[64] = {0}; - sprintf(ctbName, "%s.ctb%d", g_stConfInfo.dbName, pInfo->consumerId); - - int64_t lastPrintTime = taosGetTimestampUs(); - int64_t totalMsgLen = 0; - //int64_t timeStamp = taosGetTimestampUs(); - while (totalSendLoopTimes) { - int64_t startTs = taosGetTimestampUs(); - for (int i = 0; i < batchPerTblTimes; ++i) { - uint32_t msgsOfSql = g_stConfInfo.batchSize; - if ((i == batchPerTblTimes - 1) && (0 != remainder)) { - msgsOfSql = remainder; - } - int len = 0; - len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "insert into %s values ", ctbName); - for (int j = 0; j < msgsOfSql; j++) { - int64_t timeStamp = taosGetTimestampNs(); - len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "(%" PRId64 ", \"%s\")", timeStamp, g_payload); - sendMsgs++; - pInfo->totalProduceMsgs++; - } - - totalMsgLen += len; - pInfo->totalMsgsLen += len; - - int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE); - if (affectedRows < 0) { - taos_close(pInfo->taos); - pInfo->taos = NULL; - return NULL; - } - - affectedRowsTotal += affectedRows; - - //printf("Produce Info: affectedRows: %" PRId64 "\n", affectedRows); - } - totalSendLoopTimes -= 1; - - // calc spent time - int64_t currentTs = taosGetTimestampUs(); - int64_t delta = currentTs - startTs; - if (delta < SEND_TIME_UNIT * 1000) { - int64_t sleepLen = (int32_t)(SEND_TIME_UNIT * 1000 - delta); - //printf("sleep %" PRId64 " us, use time: %" PRId64 " us\n", sleepLen, delta); - taosUsleep((int32_t)sleepLen); - } - - currentTs = taosGetTimestampUs(); - delta = currentTs - lastPrintTime; - if (delta > 10 * 1000 * 1000) { - printf("producer[%d] info: %" PRId64 " msgs, %" PRId64 " Byte, %" PRId64 " us, totalSendLoopTimes: %d\n", - pInfo->consumerId, sendMsgs, totalMsgLen, delta, totalSendLoopTimes); - printf("producer[%d] rate: %1.f msgs/s, %1.f KB/s\n", - pInfo->consumerId, - sendMsgs * 1000.0 * 1000 / delta, - (totalMsgLen / 1024.0) / (delta / (1000*1000))); - lastPrintTime = currentTs; - sendMsgs = 0; - totalMsgLen = 0; - } - } - - printf("affectedRowsTotal: %"PRId64"\n", affectedRowsTotal); - taos_close(pInfo->taos); - pInfo->taos = NULL; - return NULL; -} - - -void printProduceInfo(int64_t start) { - int64_t totalMsgs = 0; - int64_t totalLenOfMsgs = 0; - for (int i = 0; i < g_stConfInfo.producers; i++) { - totalMsgs += g_stConfInfo.stProdThreads[i].totalProduceMsgs; - totalLenOfMsgs += g_stConfInfo.stProdThreads[i].totalMsgsLen; - } - - int64_t end = taosGetTimestampUs(); - - int64_t t = end - start; - if (0 == t) t = 1; - - double tInMs = (double)t / 1000000.0; - printf("Spent %.3f seconds to prod %" PRIu64 " msgs, %" PRIu64 " Byte\n\n", tInMs, totalMsgs, totalLenOfMsgs); - - - printf("Spent %.3f seconds to prod %" PRIu64 " msgs with %d producer(s), throughput: %.3f msgs/s, %.1f MB/s\n\n", - tInMs, totalMsgs, g_stConfInfo.producers, - (double)totalMsgs / tInMs, - (double)totalLenOfMsgs/(1024.0*1024)/tInMs); - return; -} - - -void startOmbConsume() { - TdThreadAttr thattr; - taosThreadAttrInit(&thattr); - taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - - if (0 != g_stConfInfo.producers) { - TAOS* taos = createNewTaosConnect(); - if (taos == NULL) { - taosFprintfFile(g_fp, "taos_connect() fail, can not create db, stbl, ctbl, topic!\n"); - return ; - } - - char stbName[16] = "stb"; - char ctbPrefix[16] = "ctb"; - - char sql[256] = {0}; - sprintf(sql, "drop database if exists %s", g_stConfInfo.dbName); - printf("SQL: %s\n", sql); - queryDbExec(taos, sql, NO_INSERT_TYPE); - - sprintf(sql, "create database if not exists %s precision 'ns' vgroups %d", g_stConfInfo.dbName, g_stConfInfo.producers); - printf("SQL: %s\n", sql); - queryDbExec(taos, sql, NO_INSERT_TYPE); - - sprintf(sql, "create stable %s.%s (ts timestamp, payload binary(%d)) tags (t bigint) ", g_stConfInfo.dbName, stbName, g_stConfInfo.payloadLen); - printf("SQL: %s\n", sql); - queryDbExec(taos, sql, NO_INSERT_TYPE); - - for (int i = 0; i < g_stConfInfo.producers; i++) { - sprintf(sql, "create table %s.%s%d using %s.stb tags(%d) ", g_stConfInfo.dbName, ctbPrefix, i, g_stConfInfo.dbName, i); - printf("SQL: %s\n", sql); - queryDbExec(taos, sql, NO_INSERT_TYPE); - } - - // create topic - sprintf(sql, "create topic %s as stable %s.%s", g_stConfInfo.topic, g_stConfInfo.dbName, stbName); - printf("SQL: %s\n", sql); - queryDbExec(taos, sql, NO_INSERT_TYPE); - - - int32_t producerRate = ceil(g_stConfInfo.producerRate / g_stConfInfo.producers); - - printf("==== create %d produce thread ====\n", g_stConfInfo.producers); - for (int32_t i = 0; i < g_stConfInfo.producers; ++i) { - g_stConfInfo.stProdThreads[i].consumerId = i; - g_stConfInfo.stProdThreads[i].producerRate = producerRate; - taosThreadCreate(&(g_stConfInfo.stProdThreads[i].thread), &thattr, ombProduceThreadFunc, - (void*)(&(g_stConfInfo.stProdThreads[i]))); - } - - if (0 == g_stConfInfo.numOfThread) { - int64_t start = taosGetTimestampUs(); - for (int32_t i = 0; i < g_stConfInfo.producers; i++) { - taosThreadJoin(g_stConfInfo.stProdThreads[i].thread, NULL); - taosThreadClear(&g_stConfInfo.stProdThreads[i].thread); - } - - printProduceInfo(start); - - taosFprintfFile(g_fp, "==== close tmqlog ====\n"); - taosCloseFile(&g_fp); - taos_close(taos); - return; - } - - taos_close(taos); - } - - // pthread_create one thread to consume - taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread); - for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) { - g_stConfInfo.stThreads[i].consumerId = i; - taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, ombConsumeThreadFunc, - (void*)(&(g_stConfInfo.stThreads[i]))); - } - - int64_t start = taosGetTimestampUs(); - - for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { - taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL); - taosThreadClear(&g_stConfInfo.stThreads[i].thread); - } - - int64_t end = taosGetTimestampUs(); - - int64_t totalRows = 0; - int64_t totalMsgs = 0; - int64_t totalLenOfMsgs = 0; - for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { - totalMsgs += g_stConfInfo.stThreads[i].consumeMsgCnt; - totalLenOfMsgs += g_stConfInfo.stThreads[i].consumeLen; - totalRows += g_stConfInfo.stThreads[i].consumeRowCnt; - } - - int64_t t = end - start; - if (0 == t) t = 1; - - double tInMs = (double)t / 1000000.0; - taosFprintfFile(g_fp, - "Spent %.3f seconds to poll msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/s, %.1f MB/s\n\n", - tInMs, totalMsgs, g_stConfInfo.numOfThread, - (double)(totalMsgs / tInMs), - (double)totalLenOfMsgs/(1024*1024)/tInMs); - - printf("Spent %.3f seconds to cons rows: %" PRIu64 " msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/s, %.1f MB/s\n\n", - tInMs, totalRows, totalMsgs, g_stConfInfo.numOfThread, - (double)(totalMsgs / tInMs), - (double)totalLenOfMsgs/(1024*1024)/tInMs); - - taosFprintfFile(g_fp, "==== close tmqlog ====\n"); - taosCloseFile(&g_fp); - - return; -} - - -int main(int32_t argc, char* argv[]) { - parseArgument(argc, argv); - - if (0 != strlen(g_stConfInfo.topic)) { - startOmbConsume(); - return 0; - } - - int32_t retCode = getConsumeInfo(); - if (0 != retCode) { - return -1; - } - - saveConfigToLogFile(); - - tmqSetSignalHandle(); - - TdThreadAttr thattr; - taosThreadAttrInit(&thattr); - taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - - // pthread_create one thread to consume - taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread); - for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) { - taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc, - (void*)(&(g_stConfInfo.stThreads[i]))); - } - - int64_t start = taosGetTimestampUs(); - - for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { - taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL); - taosThreadClear(&g_stConfInfo.stThreads[i].thread); - } - - int64_t end = taosGetTimestampUs(); - - int64_t totalMsgs = 0; - for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { - totalMsgs += g_stConfInfo.stThreads[i].consumeMsgCnt; - } - - int64_t t = end - start; - if (0 == t) t = 1; - - double tInMs = (double)t / 1000000.0; - taosFprintfFile(g_fp, - "Spent %.3f seconds to poll msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/second\n\n", - tInMs, totalMsgs, g_stConfInfo.numOfThread, (double)(totalMsgs / tInMs)); - - taosFprintfFile(g_fp, "==== close tmqlog ====\n"); - taosCloseFile(&g_fp); - - return 0; -} diff --git a/tests/test/c/tmq_taosx_ci.c b/tests/test/c/tmq_taosx_ci.c deleted file mode 100644 index f917b9159e..0000000000 --- a/tests/test/c/tmq_taosx_ci.c +++ /dev/null @@ -1,623 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include -#include -#include -#include "taos.h" -#include "types.h" - -static int running = 1; -TdFilePtr g_fp = NULL; -typedef struct{ - bool snapShot; - bool dropTable; - bool subTable; - int srcVgroups; - int dstVgroups; - char dir[64]; -}Config; - -Config g_conf = {0}; - -static TAOS* use_db(){ - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - if (pConn == NULL) { - return NULL; - } - - TAOS_RES* pRes = taos_query(pConn, "use db_taosx"); - if (taos_errno(pRes) != 0) { - printf("error in use db_taosx, reason:%s\n", taos_errstr(pRes)); - return NULL; - } - taos_free_result(pRes); - return pConn; -} - -static void msg_process(TAOS_RES* msg) { - printf("-----------topic-------------: %s\n", tmq_get_topic_name(msg)); - printf("db: %s\n", tmq_get_db_name(msg)); - printf("vg: %d\n", tmq_get_vgroup_id(msg)); - TAOS *pConn = use_db(); - if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { - char* result = tmq_get_json_meta(msg); - if (result) { - printf("meta result: %s\n", result); - } - if(g_fp){ - taosFprintfFile(g_fp, result); - taosFprintfFile(g_fp, "\n"); - } - - tmq_free_json_meta(result); - } - - tmq_raw_data raw = {0}; - tmq_get_raw(msg, &raw); - int32_t ret = tmq_write_raw(pConn, raw); - printf("write raw data: %s\n", tmq_err2str(ret)); - - taos_close(pConn); -} - -int buildDatabase(TAOS* pConn, TAOS_RES* pRes){ - pRes = taos_query(pConn, - "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " - "nchar(8), t4 bool)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000, \"ttt\", true)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ct0 values(1626006833400, 1, 2, 'a')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists ct1 using st1(t1) tags(2000)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table ct1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists ct2 using st1(t1) tags(NULL)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table ct2, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ct1 values(1626006833600, 3, 4, 'b')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists ct3 using st1(t1) tags(3000)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ct3 values(1626006833600, 5, 6, 'c') ct1 values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, 'ddd') ct0 values(1626006833603, 4, 3, 'hwj') ct1 values(now+5s, 23, 32, 's21ds')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table st1 add column c4 bigint"); - if (taos_errno(pRes) != 0) { - printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table st1 modify column c3 binary(64)"); - if (taos_errno(pRes) != 0) { - printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ct3 values(1626006833605, 53, 63, 'cffffffffffffffffffffffffffff', 8989898899999) (1626006833609, 51, 62, 'c333', 940)"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ct3 select * from ct1"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table st1 add tag t2 binary(64)"); - if (taos_errno(pRes) != 0) { - printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table ct3 set tag t1=5000"); - if (taos_errno(pRes) != 0) { - printf("failed to slter child table ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "delete from abc1 .ct3 where ts < 1626006833606"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - if(g_conf.dropTable){ - pRes = taos_query(pConn, "drop table ct3 ct1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop child table ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop table st1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - } - - pRes = taos_query(pConn, "create table if not exists n1(ts timestamp, c1 int, c2 nchar(4))"); - if (taos_errno(pRes) != 0) { - printf("failed to create normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 add column c3 bigint"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 modify column c2 nchar(8)"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 rename column c3 cc3"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 comment 'hello'"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 drop column c1"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into n1 values(now, 'eeee', 8989898899999) (now+9s, 'c333', 940)"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - if(g_conf.dropTable){ - pRes = taos_query(pConn, "drop table n1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - } - - pRes = taos_query(pConn, "create table jt(ts timestamp, i int) tags(t json)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table jt1 using jt tags('{\"k1\":1, \"k2\":\"hello\"}')"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table jt2 using jt tags('')"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into jt1 values(now, 1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table jt1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into jt2 values(now, 11)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - if(g_conf.dropTable){ - pRes = taos_query(pConn, - "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " - "nchar(8), t4 bool)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop table st1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - } - return 0; -} - -int buildStable(TAOS* pConn, TAOS_RES* pRes){ - pRes = taos_query(pConn, "CREATE STABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` VARCHAR(16))"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table meters, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table d0 using meters tags(1, 'San Francisco')"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table d0, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table d1 using meters tags(2, 'Beijing')"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table d1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create stream meters_summary_s into meters_summary as select _wstart, max(current) as current, groupid, location from meters partition by groupid, location interval(10m)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table meters_summary, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into d0 (ts, current) values (now, 120)"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into table d0, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - return 0; -} - -int32_t init_env() { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - if (pConn == NULL) { - return -1; - } - - TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_taosx"); - if (taos_errno(pRes) != 0) { - printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - char sql[128] = {0}; - snprintf(sql, 128, "create database if not exists db_taosx vgroups %d", g_conf.dstVgroups); - pRes = taos_query(pConn, sql); - if (taos_errno(pRes) != 0) { - printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop topic if exists topic_db"); - if (taos_errno(pRes) != 0) { - printf("error in drop topic, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop topic if exists meters_summary_t1"); - if (taos_errno(pRes) != 0) { - printf("error in drop topic, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop database if exists abc1"); - if (taos_errno(pRes) != 0) { - printf("error in drop db, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - snprintf(sql, 128, "create database if not exists abc1 vgroups %d", g_conf.srcVgroups); - pRes = taos_query(pConn, sql); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - if(g_conf.subTable){ - buildStable(pConn, pRes); - }else{ - buildDatabase(pConn, pRes); - } - - taos_close(pConn); - return 0; -} - -int32_t create_topic() { - printf("create topic\n"); - TAOS_RES* pRes; - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - if (pConn == NULL) { - return -1; - } - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - if(g_conf.subTable){ - pRes = taos_query(pConn, "create topic meters_summary_t1 with meta as stable meters_summary"); - if (taos_errno(pRes) != 0) { - printf("failed to create topic meters_summary_t1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - }else{ - pRes = taos_query(pConn, "create topic topic_db with meta as database abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to create topic topic_db, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - } - - taos_close(pConn); - return 0; -} - -void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { - printf("commit %d tmq %p param %p\n", code, tmq, param); -} - -tmq_t* build_consumer() { - tmq_conf_t* conf = tmq_conf_new(); - tmq_conf_set(conf, "group.id", "tg2"); - tmq_conf_set(conf, "client.id", "my app 1"); - tmq_conf_set(conf, "td.connect.user", "root"); - tmq_conf_set(conf, "td.connect.pass", "taosdata"); - tmq_conf_set(conf, "msg.with.table.name", "true"); - tmq_conf_set(conf, "enable.auto.commit", "true"); - tmq_conf_set(conf, "enable.heartbeat.background", "true"); - - if(g_conf.snapShot){ - tmq_conf_set(conf, "experimental.snapshot.enable", "true"); - } - - tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); - tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); - assert(tmq); - tmq_conf_destroy(conf); - return tmq; -} - -tmq_list_t* build_topic_list() { - tmq_list_t* topic_list = tmq_list_new(); - if(g_conf.subTable){ - tmq_list_append(topic_list, "meters_summary_t1"); - }else{ - tmq_list_append(topic_list, "topic_db"); - } - return topic_list; -} - -void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { - int32_t code; - - if ((code = tmq_subscribe(tmq, topics))) { - fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); - printf("subscribe err\n"); - return; - } - int32_t cnt = 0; - while (running) { - TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000); - if (tmqmessage) { - cnt++; - msg_process(tmqmessage); - taos_free_result(tmqmessage); - }else{ - break; - } - } - - code = tmq_consumer_close(tmq); - if (code) - fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); - else - fprintf(stderr, "%% Consumer closed\n"); -} - -void initLogFile() { - char f1[256] = {0}; - char f2[256] = {0}; - - if(g_conf.snapShot){ - sprintf(f1, "%s/../log/tmq_taosx_tmp_snapshot.source", g_conf.dir); - sprintf(f2, "%s/../log/tmq_taosx_tmp_snapshot.result", g_conf.dir); - }else{ - sprintf(f1, "%s/../log/tmq_taosx_tmp.source", g_conf.dir); - sprintf(f2, "%s/../log/tmq_taosx_tmp.result", g_conf.dir); - } - - TdFilePtr pFile = taosOpenFile(f1, TD_FILE_TEXT | TD_FILE_TRUNC | TD_FILE_STREAM); - if (NULL == pFile) { - fprintf(stderr, "Failed to open %s for save result\n", f1); - exit(-1); - } - g_fp = pFile; - - TdFilePtr pFile2 = taosOpenFile(f2, TD_FILE_TEXT | TD_FILE_TRUNC | TD_FILE_STREAM); - if (NULL == pFile2) { - fprintf(stderr, "Failed to open %s for save result\n", f2); - exit(-1); - } - - if(g_conf.snapShot){ - char *result[] = { - "{\"type\":\"create\",\"tableName\":\"st1\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":64},{\"name\":\"c4\",\"type\":5}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1},{\"name\":\"t2\",\"type\":8,\"length\":64}]}", - "{\"type\":\"create\",\"tableName\":\"ct0\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}", - "{\"type\":\"create\",\"tableName\":\"ct1\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}]}", - "{\"type\":\"create\",\"tableName\":\"ct2\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[]}", - "{\"type\":\"create\",\"tableName\":\"ct3\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":5000}]}", - "{\"type\":\"create\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c2\",\"type\":10,\"length\":8},{\"name\":\"cc3\",\"type\":5}],\"tags\":[]}", - "{\"type\":\"create\",\"tableName\":\"jt\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", - "{\"type\":\"create\",\"tableName\":\"jt1\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}]}", - "{\"type\":\"create\",\"tableName\":\"jt2\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]}", - }; - - for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){ - taosFprintfFile(pFile2, result[i]); - taosFprintfFile(pFile2, "\n"); - } - }else{ - char *result[] = { - "{\"type\":\"create\",\"tableName\":\"st1\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", - "{\"type\":\"create\",\"tableName\":\"ct0\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}", - "{\"type\":\"create\",\"tableName\":\"ct1\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}]}", - "{\"type\":\"create\",\"tableName\":\"ct2\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[]}", - "{\"type\":\"create\",\"tableName\":\"ct3\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":3000}]}", - "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":5,\"colName\":\"c4\",\"colType\":5}", - "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":7,\"colName\":\"c3\",\"colType\":8,\"colLength\":64}", - "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":1,\"colName\":\"t2\",\"colType\":8,\"colLength\":64}", - "{\"type\":\"alter\",\"tableName\":\"ct3\",\"tableType\":\"child\",\"alterType\":4,\"colName\":\"t1\",\"colValue\":\"5000\",\"colValueNull\":false}", - "{\"type\":\"create\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}", - "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":5,\"colName\":\"c3\",\"colType\":5}", - "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":7,\"colName\":\"c2\",\"colType\":10,\"colLength\":8}", - "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":10,\"colName\":\"c3\",\"colNewName\":\"cc3\"}", - "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":9}", - "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":6,\"colName\":\"c1\"}", - "{\"type\":\"create\",\"tableName\":\"jt\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", - "{\"type\":\"create\",\"tableName\":\"jt1\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}]}", - "{\"type\":\"create\",\"tableName\":\"jt2\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]}" - }; - - for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){ - taosFprintfFile(pFile2, result[i]); - taosFprintfFile(pFile2, "\n"); - } - } - - taosCloseFile(&pFile2); -} - -int main(int argc, char* argv[]) { - for (int32_t i = 1; i < argc; i++) { - if(strcmp(argv[i], "-c") == 0){ - strcpy(g_conf.dir, argv[++i]); - }else if(strcmp(argv[i], "-s") == 0){ - g_conf.snapShot = true; - }else if(strcmp(argv[i], "-d") == 0){ - g_conf.dropTable = true; - }else if(strcmp(argv[i], "-sv") == 0){ - g_conf.srcVgroups = atol(argv[++i]); - }else if(strcmp(argv[i], "-dv") == 0){ - g_conf.dstVgroups = atol(argv[++i]); - }else if(strcmp(argv[i], "-t") == 0){ - g_conf.subTable = true; - } - } - - printf("env init\n"); - if(strlen(g_conf.dir) != 0){ - initLogFile(); - } - - if (init_env() < 0) { - return -1; - } - create_topic(); - - tmq_t* tmq = build_consumer(); - tmq_list_t* topic_list = build_topic_list(); - basic_consume_loop(tmq, topic_list); - taosCloseFile(&g_fp); -} diff --git a/tests/tsim/CMakeLists.txt b/tests/tsim/CMakeLists.txt deleted file mode 100644 index c2cf7ac3c5..0000000000 --- a/tests/tsim/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -aux_source_directory(src TSIM_SRC) -add_executable(tsim ${TSIM_SRC}) -target_link_libraries( - tsim - PUBLIC taos_static - PUBLIC util - PUBLIC common - PUBLIC os - PUBLIC cjson -) -target_include_directories( - tsim - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) diff --git a/tests/tsim/inc/simInt.h b/tests/tsim/inc/simInt.h deleted file mode 100644 index f9e96fc67b..0000000000 --- a/tests/tsim/inc/simInt.h +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_SIM_INT_H_ -#define _TD_SIM_INT_H_ - -#include "os.h" - -#include "cJSON.h" -#include "tconfig.h" -#include "taos.h" -#include "taoserror.h" -#include "tidpool.h" -#include "tlog.h" -#include "ttimer.h" -#include "ttypes.h" -#include "tutil.h" -#include "tglobal.h" - -#define MAX_MAIN_SCRIPT_NUM 10 -#define MAX_BACKGROUND_SCRIPT_NUM 10 -#define MAX_FILE_NAME_LEN 256 -#define MAX_ERROR_LEN 1024 -#define MAX_QUERY_VALUE_LEN 1024 -#define MAX_QUERY_COL_NUM 100 -#define MAX_QUERY_ROW_NUM 100 -#define MAX_SYSTEM_RESULT_LEN 2048 -#define MAX_VAR_LEN 100 -#define MAX_VAR_NAME_LEN 32 -#define MAX_VAR_VAL_LEN 80 -#define MAX_OPT_NAME_LEN 32 -#define MAX_SIM_CMD_NAME_LEN 40 - -#ifdef LINUX -#define SUCCESS_PREFIX "\033[44;32;1m" -#define SUCCESS_POSTFIX "\033[0m" -#define FAILED_PREFIX "\033[44;31;1m" -#define FAILED_POSTFIX "\033[0m" -#else -#define SUCCESS_PREFIX "" -#define SUCCESS_POSTFIX "" -#define FAILED_PREFIX "" -#define FAILED_POSTFIX "" -#endif - -#define simFatal(...) { if (simDebugFlag & DEBUG_FATAL) { taosPrintLog("SIM FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} -#define simError(...) { if (simDebugFlag & DEBUG_ERROR) { taosPrintLog("SIM ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} -#define simWarn(...) { if (simDebugFlag & DEBUG_WARN) { taosPrintLog("SIM WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} -#define simInfo(...) { if (simDebugFlag & DEBUG_INFO) { taosPrintLog("SIM ", DEBUG_INFO, 255, __VA_ARGS__); }} -#define simDebug(...) { if (simDebugFlag & DEBUG_DEBUG) { taosPrintLog("SIM ", DEBUG_DEBUG, simDebugFlag, __VA_ARGS__); }} -#define simTrace(...) { if (simDebugFlag & DEBUG_TRACE) { taosPrintLog("SIM ", DEBUG_TRACE, simDebugFlag, __VA_ARGS__); }} - -enum { SIM_SCRIPT_TYPE_MAIN, SIM_SCRIPT_TYPE_BACKGROUND }; - -enum { - SIM_CMD_EXP, - SIM_CMD_IF, - SIM_CMD_ELIF, - SIM_CMD_ELSE, - SIM_CMD_ENDI, - SIM_CMD_WHILE, - SIM_CMD_ENDW, - SIM_CMD_SWITCH, - SIM_CMD_CASE, - SIM_CMD_DEFAULT, - SIM_CMD_CONTINUE, - SIM_CMD_BREAK, - SIM_CMD_ENDS, - SIM_CMD_SLEEP, - SIM_CMD_GOTO, - SIM_CMD_RUN, - SIM_CMD_RUN_BACK, - SIM_CMD_PRINT, - SIM_CMD_SYSTEM, - SIM_CMD_SYSTEM_CONTENT, - SIM_CMD_SQL, - SIM_CMD_SQL_ERROR, - SIM_CMD_SQL_SLOW, - SIM_CMD_RESTFUL, - SIM_CMD_TEST, - SIM_CMD_RETURN, - SIM_CMD_LINE_INSERT, - SIM_CMD_LINE_INSERT_ERROR, - SIM_CMD_END -}; - -enum { - SQL_JUMP_FALSE, - SQL_JUMP_TRUE, -}; - -struct _script_t; -typedef struct _cmd_t { - int16_t cmdno; - int16_t nlen; - char name[MAX_SIM_CMD_NAME_LEN]; - bool (*parseCmd)(char *, struct _cmd_t *, int32_t); - bool (*executeCmd)(struct _script_t *script, char *option); - struct _cmd_t *next; -} SCommand; - -typedef struct { - int16_t cmdno; - int16_t jump; // jump position - int16_t errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag - // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */ - int16_t lineNum; // correspodning line number in original file - int32_t optionOffset; // relative option offset -} SCmdLine; - -typedef struct _var_t { - char varName[MAX_VAR_NAME_LEN]; - char varValue[MAX_VAR_VAL_LEN]; - char varNameLen; -} SVariable; - -typedef struct _script_t { - int32_t type; - bool killed; - void *taos; - char rows[12]; // number of rows data retrieved - char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN]; // query results - char system_exit_code[12]; - char system_ret_content[MAX_SYSTEM_RESULT_LEN]; - int32_t varLen; - int32_t linePos; // current cmd position - int32_t numOfLines; // number of lines in the script - int32_t bgScriptLen; - char fileName[MAX_FILE_NAME_LEN]; // script file name - char error[MAX_ERROR_LEN]; - char *optionBuffer; - SCmdLine *lines; // command list - SVariable variables[MAX_VAR_LEN]; - TdThread bgPid; - char auth[128]; - struct _script_t *bgScripts[MAX_BACKGROUND_SCRIPT_NUM]; -} SScript; - -extern SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; -extern SCommand simCmdList[]; -extern int32_t simScriptPos; -extern int32_t simScriptSucced; -extern int32_t simDebugFlag; -extern char simScriptDir[]; -extern bool abortExecution; -extern bool useMultiProcess; -extern bool useValgrind; - -SScript *simParseScript(char *fileName); -SScript *simProcessCallOver(SScript *script); -void *simExecuteScript(void *script); -void simInitsimCmdList(); -bool simSystemInit(); -void simSystemCleanUp(); -char *simGetVariable(SScript *script, char *varName, int32_t varLen); -bool simExecuteExpCmd(SScript *script, char *option); -bool simExecuteTestCmd(SScript *script, char *option); -bool simExecuteGotoCmd(SScript *script, char *option); -bool simExecuteRunCmd(SScript *script, char *option); -bool simExecuteRunBackCmd(SScript *script, char *option); -bool simExecuteSystemCmd(SScript *script, char *option); -bool simExecuteSystemContentCmd(SScript *script, char *option); -bool simExecutePrintCmd(SScript *script, char *option); -bool simExecuteSleepCmd(SScript *script, char *option); -bool simExecuteReturnCmd(SScript *script, char *option); -bool simExecuteSqlCmd(SScript *script, char *option); -bool simExecuteSqlErrorCmd(SScript *script, char *rest); -bool simExecuteSqlSlowCmd(SScript *script, char *option); -bool simExecuteRestfulCmd(SScript *script, char *rest); -bool simExecuteLineInsertCmd(SScript *script, char *option); -bool simExecuteLineInsertErrorCmd(SScript *script, char *option); -void simVisuallizeOption(SScript *script, char *src, char *dst); - -#endif /*_TD_SIM_INT_H_*/ \ No newline at end of file diff --git a/tests/tsim/inc/simParse.h b/tests/tsim/inc/simParse.h deleted file mode 100644 index 2a683f9561..0000000000 --- a/tests/tsim/inc/simParse.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_SIM_PARSE_H_ -#define _TD_SIM_PARSE_H_ - -#include "simInt.h" - -#define MAX_NUM_CMD 64 -#define MAX_NUM_LABLES 100 -#define MAX_LABEL_LEN 40 -#define MAX_NUM_BLOCK 100 -#define MAX_NUM_JUMP 100 -#define MAX_LINE_LEN 3000 -#define MAX_CMD_LINES 2048 -#define MAX_OPTION_BUFFER 64000 - -enum { - BLOCK_IF, - BLOCK_WHILE, - BLOCK_SWITCH, -}; - -/* label stack */ -typedef struct { - int8_t top; /* number of labels */ - int16_t pos[MAX_NUM_LABLES]; /* the position of the label */ - char label[MAX_NUM_LABLES][MAX_LABEL_LEN]; /* name of the label */ -} SLabel; - -/* block definition */ -typedef struct { - int8_t top; /* the number of blocks stacked */ - char type[MAX_NUM_BLOCK]; /* the block type */ - int16_t *pos[MAX_NUM_BLOCK]; /* position of the jump for if/elif/case */ - int16_t back[MAX_NUM_BLOCK]; /* go back, endw and continue */ - char numJump[MAX_NUM_BLOCK]; - int16_t *jump[MAX_NUM_BLOCK][MAX_NUM_JUMP]; /* break or elif */ - char sexp[MAX_NUM_BLOCK][40]; /*switch expression */ - char sexpLen[MAX_NUM_BLOCK]; /*switch expression length */ -} SBlock; - -bool simParseExpression(char *token, int32_t lineNum); - -#endif /*_TD_SIM_PARSE_H_*/ \ No newline at end of file diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c deleted file mode 100644 index 16732ff9a1..0000000000 --- a/tests/tsim/src/simExe.c +++ /dev/null @@ -1,1016 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "simInt.h" - -void simLogSql(char *sql, bool useSharp) { - static TdFilePtr pFile = NULL; - char filename[256]; - sprintf(filename, "%s/sim.sql", simScriptDir); - if (pFile == NULL) { - // fp = fopen(filename, "w"); - pFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); - if (pFile == NULL) { - fprintf(stderr, "ERROR: failed to open file: %s\n", filename); - return; - } - } - if (useSharp) { - taosFprintfFile(pFile, "# %s;\n", sql); - } else { - taosFprintfFile(pFile, "%s;\n", sql); - } - - taosFsyncFile(pFile); -} - -char *simParseArbitratorName(char *varName) { - static char hostName[140]; -#ifdef WINDOWS - taosGetFqdn(hostName); - sprintf(&hostName[strlen(hostName)], ":%d", 8000); -#else - sprintf(hostName, "%s:%d", "localhost", 8000); -#endif - return hostName; -} - -char *simParseHostName(char *varName) { - static char hostName[140]; -#ifdef WINDOWS - hostName[0] = '\"'; - taosGetFqdn(&hostName[1]); - int strEndIndex = strlen(hostName); - hostName[strEndIndex] = '\"'; - hostName[strEndIndex + 1] = '\0'; -#else - sprintf(hostName, "%s", "localhost"); -#endif - return hostName; -} - -static void simFindFirstNum(const char *begin, int32_t beginLen, int32_t *num) { - if (beginLen <= 5) { - *num = 0; - } else { - *num = atoi(begin + 5); - } -} - -static void simFindSecondNum(const char *begin, int32_t beginLen, int32_t *num) { - const char *number = strstr(begin, "]["); - if (number == NULL) { - *num = 0; - } else { - *num = atoi(number + 2); - } -} - -static void simFindFirstKeyVal(const char *begin, int32_t beginLen, char *key, int32_t keyLen) { - key[0] = 0; - for (int32_t i = 5; i < beginLen && i - 5 < keyLen; ++i) { - if (begin[i] != 0 && begin[i] != ']' && begin[i] != ')') { - key[i - 5] = begin[i]; - } - } -} - -static void simFindSecondKeyNum(const char *begin, int32_t beginLen, int32_t *num) { - const char *number = strstr(begin, ")["); - if (number == NULL) { - *num = 0; - } else { - *num = atoi(number + 2); - } -} - -char *simGetVariable(SScript *script, char *varName, int32_t varLen) { - if (strncmp(varName, "hostname", 8) == 0) { - return simParseHostName(varName); - } - - if (strncmp(varName, "arbitrator", 10) == 0) { - return simParseArbitratorName(varName); - } - - if (strncmp(varName, "error", varLen) == 0) return script->error; - - if (strncmp(varName, "rows", varLen) == 0) return script->rows; - - if (strncmp(varName, "system_exit", varLen) == 0) return script->system_exit_code; - - if (strncmp(varName, "system_content", varLen) == 0) return script->system_ret_content; - - if (strncmp(varName, "data", 4) == 0) { - if (varLen < 6) { - return "null"; - } - - int32_t row = 0; - int32_t col = 0; - char keyVal[1024] = {0}; - int32_t keyLen = 1024; - - if (varName[4] == '[') { - // $data[0][1] - simFindFirstNum(varName, varLen, &row); - simFindSecondNum(varName, varLen, &col); - if (row < 0 || row >= MAX_QUERY_ROW_NUM) { - return "null"; - } - if (col < 0 || col >= MAX_QUERY_COL_NUM) { - return "null"; - } - simDebug("script:%s, data[%d][%d]=%s", script->fileName, row, col, script->data[row][col]); - return script->data[row][col]; - } else if (varName[4] == '(') { - // $data(db)[0] - simFindFirstKeyVal(varName, varLen, keyVal, keyLen); - simFindSecondKeyNum(varName, varLen, &col); - for (int32_t i = 0; i < MAX_QUERY_ROW_NUM; ++i) { - if (strncmp(keyVal, script->data[i][0], keyLen) == 0) { - simDebug("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]); - return script->data[i][col]; - } - } - } else if (varName[5] == '_') { - // data2_db - int32_t col = varName[4] - '0'; - if (col < 0 || col >= MAX_QUERY_COL_NUM) { - return "null"; - } - - char *keyName; - int32_t keyLen; - paGetToken(varName + 6, &keyName, &keyLen); - - for (int32_t i = 0; i < MAX_QUERY_ROW_NUM; ++i) { - if (strncmp(keyName, script->data[i][0], keyLen) == 0) { - simDebug("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]); - return script->data[i][col]; - } - } - return "null"; - } else if (varName[6] == '_') { - // data21_db - int32_t col = (varName[4] - '0') * 10 + (varName[5] - '0'); - if (col < 0 || col >= MAX_QUERY_COL_NUM) { - return "null"; - } - - char *keyName; - int32_t keyLen; - paGetToken(varName + 7, &keyName, &keyLen); - - for (int32_t i = 0; i < MAX_QUERY_ROW_NUM; ++i) { - if (strncmp(keyName, script->data[i][0], keyLen) == 0) { - simTrace("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]); - return script->data[i][col]; - } - } - return "null"; - } else { - // $data00 - int32_t row = varName[4] - '0'; - int32_t col = varName[5] - '0'; - if (row < 0 || row >= MAX_QUERY_ROW_NUM) { - return "null"; - } - if (col < 0 || col >= MAX_QUERY_COL_NUM) { - return "null"; - } - - simDebug("script:%s, data[%d][%d]=%s", script->fileName, row, col, script->data[row][col]); - return script->data[row][col]; - } - - return "null"; - } - - for (int32_t i = 0; i < script->varLen; ++i) { - SVariable *var = &script->variables[i]; - if (var->varNameLen != varLen) { - continue; - } - if (strncmp(varName, var->varName, varLen) == 0) { - return var->varValue; - } - } - - if (script->varLen >= MAX_VAR_LEN) { - simError("script:%s, too many varialbes:%d", script->fileName, script->varLen); - exit(0); - } - - SVariable *var = &script->variables[script->varLen]; - script->varLen++; - strncpy(var->varName, varName, varLen); - var->varNameLen = varLen; - var->varValue[0] = 0; - return var->varValue; -} - -int32_t simExecuteExpression(SScript *script, char *exp) { - char *op1, *op2, *var1, *var2, *var3, *rest; - int32_t op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1; - char t0[1024], t1[1024], t2[1024], t3[2048]; - int32_t result; - - rest = paGetToken(exp, &var1, &var1Len); - rest = paGetToken(rest, &op1, &op1Len); - rest = paGetToken(rest, &var2, &var2Len); - rest = paGetToken(rest, &op2, &op2Len); - - if (var1[0] == '$') - strcpy(t0, simGetVariable(script, var1 + 1, var1Len - 1)); - else { - memcpy(t0, var1, var1Len); - t0[var1Len] = 0; - } - - if (var2[0] == '$') - strcpy(t1, simGetVariable(script, var2 + 1, var2Len - 1)); - else { - memcpy(t1, var2, var2Len); - t1[var2Len] = 0; - } - - if (op2Len != 0) { - rest = paGetToken(rest, &var3, &var3Len); - - if (var3[0] == '$') - strcpy(t2, simGetVariable(script, var3 + 1, var3Len - 1)); - else { - memcpy(t2, var3, var3Len); - t2[var3Len] = 0; - } - - if (op2[0] == '+') { - sprintf(t3, "%lld", atoll(t1) + atoll(t2)); - } else if (op2[0] == '-') { - sprintf(t3, "%lld", atoll(t1) - atoll(t2)); - } else if (op2[0] == '*') { - sprintf(t3, "%lld", atoll(t1) * atoll(t2)); - } else if (op2[0] == '/') { - sprintf(t3, "%lld", atoll(t1) / atoll(t2)); - } else if (op2[0] == '.') { - sprintf(t3, "%s%s", t1, t2); - } - } else { - strcpy(t3, t1); - } - - result = 0; - - if (op1Len == 1) { - if (op1[0] == '=') { - strcpy(simGetVariable(script, var1 + 1, var1Len - 1), t3); - } else if (op1[0] == '<') { - val0 = atoi(t0); - val1 = atoi(t3); - if (val0 >= val1) result = -1; - } else if (op1[0] == '>') { - val0 = atoi(t0); - val1 = atoi(t3); - if (val0 <= val1) result = -1; - } - } else { - if (op1[0] == '=' && op1[1] == '=') { - if (strcmp(t0, t3) != 0) result = -1; - } else if (op1[0] == '!' && op1[1] == '=') { - if (strcmp(t0, t3) == 0) result = -1; - } else if (op1[0] == '<' && op1[1] == '=') { - val0 = atoi(t0); - val1 = atoi(t3); - if (val0 > val1) result = -1; - } else if (op1[0] == '>' && op1[1] == '=') { - val0 = atoi(t0); - val1 = atoi(t3); - if (val0 < val1) result = -1; - } - } - - return result; -} - -bool simExecuteExpCmd(SScript *script, char *option) { - simExecuteExpression(script, option); - script->linePos++; - return true; -} - -bool simExecuteTestCmd(SScript *script, char *option) { - int32_t result; - result = simExecuteExpression(script, option); - - if (result >= 0) - script->linePos++; - else - script->linePos = script->lines[script->linePos].jump; - - return true; -} - -bool simExecuteGotoCmd(SScript *script, char *option) { - script->linePos = script->lines[script->linePos].jump; - return true; -} - -bool simExecuteRunCmd(SScript *script, char *option) { - char *fileName = option; - if (fileName == NULL || strlen(fileName) == 0) { - sprintf(script->error, "lineNum:%d. script file is null", script->lines[script->linePos].lineNum); - return false; - } - - SScript *newScript = simParseScript(option); - if (newScript == NULL) { - sprintf(script->error, "lineNum:%d. parse file:%s error", script->lines[script->linePos].lineNum, fileName); - return false; - } - - simInfo("script:%s, start to execute", newScript->fileName); - - newScript->type = SIM_SCRIPT_TYPE_MAIN; - simScriptPos++; - simScriptList[simScriptPos] = newScript; - - script->linePos++; - return true; -} - -bool simExecuteRunBackCmd(SScript *script, char *option) { - char *fileName = option; - if (fileName == NULL || strlen(fileName) == 0) { - sprintf(script->error, "lineNum:%d. script file is null", script->lines[script->linePos].lineNum); - return false; - } - - SScript *newScript = simParseScript(option); - if (newScript == NULL) { - sprintf(script->error, "lineNum:%d. parse file:%s error", script->lines[script->linePos].lineNum, fileName); - return false; - } - - newScript->type = SIM_SCRIPT_TYPE_BACKGROUND; - script->bgScripts[script->bgScriptLen++] = newScript; - simInfo("script:%s, start to execute in background,", newScript->fileName); - - if (taosThreadCreate(&newScript->bgPid, NULL, simExecuteScript, (void *)newScript) != 0) { - sprintf(script->error, "lineNum:%d. create background thread failed", script->lines[script->linePos].lineNum); - return false; - } else { - simDebug("script:%s, background thread:0x%08" PRIx64 " is created", newScript->fileName, - taosGetPthreadId(newScript->bgPid)); - } - - script->linePos++; - return true; -} - -bool simReplaceStr(char *buf, char *src, char *dst) { - bool replaced = false; - char *begin = strstr(buf, src); - if (begin != NULL) { - int32_t srcLen = (int32_t)strlen(src); - int32_t dstLen = (int32_t)strlen(dst); - int32_t interval = (dstLen - srcLen); - int32_t remainLen = (int32_t)strlen(buf); - char *end = buf + remainLen; - *(end + interval) = 0; - - for (char *p = end; p >= begin; p--) { - *(p + interval) = *p; - } - - memcpy(begin, dst, dstLen); - replaced = true; - } - - simInfo("system cmd is %s", buf); - return replaced; -} - -bool simExecuteSystemCmd(SScript *script, char *option) { - char buf[4096] = {0}; - bool replaced = false; - -#ifndef WINDOWS - sprintf(buf, "cd %s; ", simScriptDir); - simVisuallizeOption(script, option, buf + strlen(buf)); -#else - sprintf(buf, "%s", simScriptDir); - simVisuallizeOption(script, option, buf + strlen(buf)); - simReplaceStr(buf, ".sh", ".bat"); -#endif - - if (useMultiProcess) { - simReplaceStr(buf, "deploy.sh", "deploy.sh -m"); - } - - if (useValgrind) { - replaced = simReplaceStr(buf, "exec.sh", "exec.sh -v"); - } - - simLogSql(buf, true); - int32_t code = system(buf); - int32_t repeatTimes = 0; - while (code < 0) { - simError("script:%s, failed to execute %s , code %d, errno:%d %s, repeatTimes:%d", script->fileName, buf, code, - errno, strerror(errno), repeatTimes); - taosMsleep(1000); - taosDflSignal(SIGCHLD); - if (repeatTimes++ >= 10) { - exit(0); - } - } - - sprintf(script->system_exit_code, "%d", code); - script->linePos++; - if (replaced && strstr(buf, "start") != NULL) { - simInfo("====> startup is slow in valgrind mode, so sleep 5 seconds after exec.sh -s start"); - taosMsleep(5000); - } - - return true; -} - -void simStoreSystemContentResult(SScript *script, char *filename) { - memset(script->system_ret_content, 0, MAX_SYSTEM_RESULT_LEN); - - TdFilePtr pFile; - // if ((fd = fopen(filename, "r")) != NULL) { - if ((pFile = taosOpenFile(filename, TD_FILE_READ)) != NULL) { - taosReadFile(pFile, script->system_ret_content, MAX_SYSTEM_RESULT_LEN - 1); - int32_t len = strlen(script->system_ret_content); - for (int32_t i = 0; i < len; ++i) { - if (script->system_ret_content[i] == '\n' || script->system_ret_content[i] == '\r') { - script->system_ret_content[i] = 0; - } - } - taosCloseFile(&pFile); - char rmCmd[MAX_FILE_NAME_LEN] = {0}; - sprintf(rmCmd, "rm -f %s", filename); -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-result" - system(rmCmd); -#pragma GCC diagnostic pop - } -} - -bool simExecuteSystemContentCmd(SScript *script, char *option) { - char buf[4096] = {0}; - char buf1[4096 + 512] = {0}; - char filename[400] = {0}; - sprintf(filename, "%s" TD_DIRSEP "%s.tmp", simScriptDir, script->fileName); - -#ifdef WINDOWS - sprintf(buf, "cd %s && ", simScriptDir); - simVisuallizeOption(script, option, buf + strlen(buf)); - sprintf(buf1, "%s > %s 2>nul", buf, filename); -#else - sprintf(buf, "cd %s; ", simScriptDir); - simVisuallizeOption(script, option, buf + strlen(buf)); - sprintf(buf1, "%s > %s 2>/dev/null", buf, filename); -#endif - - sprintf(script->system_exit_code, "%d", system(buf1)); - simStoreSystemContentResult(script, filename); - - script->linePos++; - return true; -} - -bool simExecutePrintCmd(SScript *script, char *rest) { - char buf[65536]; - - simVisuallizeOption(script, rest, buf); - rest = buf; - - simInfo("script:%s, %s", script->fileName, rest); - script->linePos++; - return true; -} - -bool simExecuteSleepCmd(SScript *script, char *option) { - int32_t delta; - char buf[1024]; - - simVisuallizeOption(script, option, buf); - option = buf; - - delta = atoi(option); - if (delta <= 0) delta = 5; - - simInfo("script:%s, sleep %dms begin", script->fileName, delta); - taosMsleep(delta); - simInfo("script:%s, sleep %dms finished", script->fileName, delta); - - char sleepStr[32] = {0}; - sprintf(sleepStr, "sleep %d", delta); - simLogSql(sleepStr, true); - - script->linePos++; - return true; -} - -bool simExecuteReturnCmd(SScript *script, char *option) { - char buf[1024]; - - simVisuallizeOption(script, option, buf); - option = buf; - - int32_t ret = 1; - if (option && option[0] != 0) ret = atoi(option); - - if (ret < 0) { - sprintf(script->error, "lineNum:%d. error return %s", script->lines[script->linePos].lineNum, option); - return false; - } else { - simInfo("script:%s, return cmd execute with:%d", script->fileName, ret); - script->linePos = script->numOfLines; - } - - script->linePos++; - return true; -} - -void simVisuallizeOption(SScript *script, char *src, char *dst) { - char *var, *token, *value; - int32_t dstLen, srcLen, tokenLen; - - dst[0] = 0, dstLen = 0; - - while (1) { - var = strchr(src, '$'); - if (var == NULL) break; - if (var && ((var - src - 1) > 0) && *(var - 1) == '\\') { - srcLen = (int32_t)(var - src - 1); - memcpy(dst + dstLen, src, srcLen); - dstLen += srcLen; - src = var; - break; - } - - srcLen = (int32_t)(var - src); - memcpy(dst + dstLen, src, srcLen); - dstLen += srcLen; - - src = paGetToken(var + 1, &token, &tokenLen); - value = simGetVariable(script, token, tokenLen); - - strcpy(dst + dstLen, value); - dstLen += (int32_t)strlen(value); - } - - strcpy(dst + dstLen, src); -} - -void simCloseNativeConnect(SScript *script) { - if (script->taos == NULL) return; - - simDebug("script:%s, taos:%p closed", script->fileName, script->taos); - taos_close(script->taos); - - script->taos = NULL; -} - -void simCloseTaosdConnect(SScript *script) { simCloseNativeConnect(script); } - -bool simCreateNativeConnect(SScript *script, char *user, char *pass) { - simCloseTaosdConnect(script); - void *taos = NULL; - for (int32_t attempt = 0; attempt < 10; ++attempt) { - if (abortExecution) { - script->killed = true; - return false; - } - - taos = taos_connect(NULL, user, pass, NULL, 0); - if (taos == NULL) { - simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), - attempt); - taosMsleep(1000); - } else { - simDebug("script:%s, user:%s connect taosd successed, attempt:%d", script->fileName, user, attempt); - break; - } - } - - if (taos == NULL) { - sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum, - taos_errstr(NULL)); - return false; - } - - script->taos = taos; - simDebug("script:%s, connect taosd successed, taos:%p", script->fileName, taos); - - return true; -} - -bool simCreateTaosdConnect(SScript *script, char *rest) { - char *user = TSDB_DEFAULT_USER; - char *token; - int32_t tokenLen; - rest = paGetToken(rest, &token, &tokenLen); - rest = paGetToken(rest, &token, &tokenLen); - if (tokenLen != 0) { - user = token; - } - - return simCreateNativeConnect(script, user, TSDB_DEFAULT_PASS); -} - -bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { - char timeStr[30] = {0}; - time_t tt; - struct tm tp; - SCmdLine *line = &script->lines[script->linePos]; - int32_t ret = -1; - - TAOS_RES *pSql = NULL; - - for (int32_t attempt = 0; attempt < 10; ++attempt) { - if (abortExecution) { - script->killed = true; - return false; - } - - simLogSql(rest, false); - pSql = taos_query(script->taos, rest); - ret = taos_errno(pSql); - - if (ret == TSDB_CODE_MND_STB_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) { - simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, - tstrerror(ret)); - ret = 0; - break; - } else if (ret != 0) { - simDebug("script:%s, taos:%p, %s failed, ret:%d:%s, error:%s", script->fileName, script->taos, rest, ret & 0XFFFF, - tstrerror(ret), taos_errstr(pSql)); - - if (line->errorJump == SQL_JUMP_TRUE) { - script->linePos = line->jump; - taos_free_result(pSql); - return true; - } - taosMsleep(1000); - } else { - break; - } - - taos_free_result(pSql); - } - - if (ret) { - sprintf(script->error, "lineNum:%d. sql:%s failed, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, tstrerror(ret)); - return false; - } - - int32_t numOfRows = 0; - int32_t num_fields = taos_field_count(pSql); - if (num_fields != 0) { - if (pSql == NULL) { - simDebug("script:%s, taos:%p, %s failed, result is null", script->fileName, script->taos, rest); - if (line->errorJump == SQL_JUMP_TRUE) { - script->linePos = line->jump; - return true; - } - - sprintf(script->error, "lineNum:%d. result set null, sql:%s", line->lineNum, rest); - return false; - } - - TAOS_ROW row; - - while ((row = taos_fetch_row(pSql))) { - if (numOfRows < MAX_QUERY_ROW_NUM) { - TAOS_FIELD *fields = taos_fetch_fields(pSql); - int32_t *length = taos_fetch_lengths(pSql); - - for (int32_t i = 0; i < num_fields; i++) { - char *value = NULL; - if (i < MAX_QUERY_COL_NUM) { - value = script->data[numOfRows][i]; - } - if (value == NULL) { - continue; - } - - if (row[i] == 0) { - strcpy(value, TSDB_DATA_NULL_STR); - continue; - } - - switch (fields[i].type) { - case TSDB_DATA_TYPE_BOOL: - sprintf(value, "%s", ((((int32_t)(*((char *)row[i]))) == 1) ? "1" : "0")); - break; - case TSDB_DATA_TYPE_TINYINT: - sprintf(value, "%d", *((int8_t *)row[i])); - break; - case TSDB_DATA_TYPE_UTINYINT: - sprintf(value, "%u", *((uint8_t *)row[i])); - break; - case TSDB_DATA_TYPE_SMALLINT: - sprintf(value, "%d", *((int16_t *)row[i])); - break; - case TSDB_DATA_TYPE_USMALLINT: - sprintf(value, "%u", *((uint16_t *)row[i])); - break; - case TSDB_DATA_TYPE_INT: - sprintf(value, "%d", *((int32_t *)row[i])); - break; - case TSDB_DATA_TYPE_UINT: - sprintf(value, "%u", *((uint32_t *)row[i])); - break; - case TSDB_DATA_TYPE_BIGINT: - sprintf(value, "%" PRId64, *((int64_t *)row[i])); - break; - case TSDB_DATA_TYPE_UBIGINT: - sprintf(value, "%" PRIu64, *((uint64_t *)row[i])); - break; - case TSDB_DATA_TYPE_FLOAT: - sprintf(value, "%.5f", GET_FLOAT_VAL(row[i])); - break; - case TSDB_DATA_TYPE_DOUBLE: - sprintf(value, "%.9lf", GET_DOUBLE_VAL(row[i])); - break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - if (length[i] < 0 || length[i] > 1 << 20) { - fprintf(stderr, "Invalid length(%d) of BINARY or NCHAR\n", length[i]); - exit(-1); - } - - memset(value, 0, MAX_QUERY_VALUE_LEN); - memcpy(value, row[i], length[i]); - value[length[i]] = 0; - // snprintf(value, fields[i].bytes, "%s", (char *)row[i]); - break; - case TSDB_DATA_TYPE_TIMESTAMP: { - int32_t precision = taos_result_precision(pSql); - if (precision == TSDB_TIME_PRECISION_MILLI) { - tt = (*(int64_t *)row[i]) / 1000; - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - tt = (*(int64_t *)row[i]) / 1000000; - } else { - tt = (*(int64_t *)row[i]) / 1000000000; - } - - taosLocalTime(&tt, &tp); - strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", &tp); - if (precision == TSDB_TIME_PRECISION_MILLI) { - sprintf(value, "%s.%03d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000)); - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - sprintf(value, "%s.%06d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000000)); - } else { - sprintf(value, "%s.%09d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000000000)); - } - - break; - } - default: - break; - } // end of switch - } // end of for - } // end of if - numOfRows++; - if (isSlow && numOfRows % 100 == 0) { - taosMsleep(200); - } - if (numOfRows > 2000000000) { - simError("script:%s, too many rows return from query", script->fileName); - break; - } - } - - } else { - numOfRows = taos_affected_rows(pSql); - } - - taos_free_result(pSql); - sprintf(script->rows, "%d", numOfRows); - - script->linePos++; - return true; -} - -bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) { - char buf[3000]; - SCmdLine *line = &script->lines[script->linePos]; - - simVisuallizeOption(script, rest, buf); - rest = buf; - - simDebug("script:%s, exec:%s", script->fileName, rest); - strcpy(script->rows, "-1"); - for (int32_t row = 0; row < MAX_QUERY_ROW_NUM; ++row) { - for (int32_t col = 0; col < MAX_QUERY_COL_NUM; ++col) { - strcpy(script->data[row][col], "null"); - } - } - - if (strncmp(rest, "connect", 7) == 0) { - if (!simCreateTaosdConnect(script, rest)) { - return false; - } - script->linePos++; - return true; - } - - if (script->taos == NULL) { - if (!simCreateTaosdConnect(script, "connect root")) { - if (line->errorJump == SQL_JUMP_TRUE) { - script->linePos = line->jump; - return true; - } - return false; - } - } - - if (strncmp(rest, "close", 5) == 0) { - simCloseTaosdConnect(script); - script->linePos++; - return true; - } - - return simExecuteNativeSqlCommand(script, rest, isSlow); -} - -bool simExecuteSqlCmd(SScript *script, char *rest) { - bool isSlow = false; - return simExecuteSqlImpCmd(script, rest, isSlow); -} - -bool simExecuteSqlSlowCmd(SScript *script, char *rest) { - bool isSlow = true; - return simExecuteSqlImpCmd(script, rest, isSlow); -} - -bool simExecuteRestfulCmd(SScript *script, char *rest) { - TdFilePtr pFile = NULL; - char filename[256]; - sprintf(filename, "%s/tmp.sql", simScriptDir); - // fp = fopen(filename, "w"); - pFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); - if (pFile == NULL) { - fprintf(stderr, "ERROR: failed to open file: %s\n", filename); - return false; - } - - char db[64] = {0}; - char tb[64] = {0}; - char gzip[32] = {0}; - int32_t ts; - int32_t times; - sscanf(rest, "%s %s %d %d %s", db, tb, &ts, ×, gzip); - - taosFprintfFile(pFile, "insert into %s.%s values ", db, tb); - for (int32_t i = 0; i < times; ++i) { - taosFprintfFile(pFile, "(%d000, %d)", ts + i, ts); - } - taosFprintfFile(pFile, " \n"); - taosFsyncFile(pFile); - taosCloseFile(&pFile); - - char cmd[1024] = {0}; - if (strcmp(gzip, "gzip") == 0) { - sprintf(cmd, - "curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' --header " - "--compressed --data-ascii @%s 127.0.0.1:7111/rest/sql", - filename); - } else { - sprintf(cmd, - "curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' --header " - "'Transfer-Encoding: chunked' --data-ascii @%s 127.0.0.1:7111/rest/sql", - filename); - } - - return simExecuteSystemCmd(script, cmd); -} - -bool simExecuteSqlErrorCmd(SScript *script, char *rest) { - char buf[3000]; - SCmdLine *line = &script->lines[script->linePos]; - - simVisuallizeOption(script, rest, buf); - rest = buf; - - simDebug("script:%s, exec:%s", script->fileName, rest); - strcpy(script->rows, "-1"); - for (int32_t row = 0; row < MAX_QUERY_ROW_NUM; ++row) { - for (int32_t col = 0; col < MAX_QUERY_COL_NUM; ++col) { - strcpy(script->data[row][col], "null"); - } - } - - if (strncmp(rest, "connect", 7) == 0) { - if (!simCreateTaosdConnect(script, rest)) { - return false; - } - script->linePos++; - return true; - } - - if (script->taos == NULL) { - if (!simCreateTaosdConnect(script, "connect root")) { - if (line->errorJump == SQL_JUMP_TRUE) { - script->linePos = line->jump; - return true; - } - return false; - } - } - - if (strncmp(rest, "close", 5) == 0) { - simCloseTaosdConnect(script); - script->linePos++; - return true; - } - - TAOS_RES *pSql = pSql = taos_query(script->taos, rest); - int32_t ret = taos_errno(pSql); - taos_free_result(pSql); - - if (ret != TSDB_CODE_SUCCESS) { - simDebug("script:%s, taos:%p, %s execute, expect failed, so success, ret:%d:%s", script->fileName, script->taos, - rest, ret & 0XFFFF, tstrerror(ret)); - script->linePos++; - return true; - } - - sprintf(script->error, "lineNum:%d. sql:%s expect failed, but success, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, - tstrerror(ret)); - - return false; -} - -bool simExecuteLineInsertCmd(SScript *script, char *rest) { - char buf[TSDB_MAX_BINARY_LEN]; - - simVisuallizeOption(script, rest, buf); - rest = buf; - - SCmdLine *line = &script->lines[script->linePos]; - - simInfo("script:%s, %s", script->fileName, rest); - simLogSql(buf, true); - char *lines[] = {rest}; -#if 0 - int32_t ret = taos_insert_lines(script->taos, lines, 1); -#else - int32_t ret = 0; -#endif - if (ret == TSDB_CODE_SUCCESS) { - simDebug("script:%s, taos:%p, %s executed. success.", script->fileName, script->taos, rest); - script->linePos++; - return true; - } else { - sprintf(script->error, "lineNum: %d. line: %s failed, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, - tstrerror(ret)); - return false; - } -} - -bool simExecuteLineInsertErrorCmd(SScript *script, char *rest) { - char buf[TSDB_MAX_BINARY_LEN]; - - simVisuallizeOption(script, rest, buf); - rest = buf; - - SCmdLine *line = &script->lines[script->linePos]; - - simInfo("script:%s, %s", script->fileName, rest); - simLogSql(buf, true); - char *lines[] = {rest}; -#if 0 - int32_t ret = taos_insert_lines(script->taos, lines, 1); -#else - int32_t ret = 0; -#endif - if (ret == TSDB_CODE_SUCCESS) { - sprintf(script->error, "script:%s, taos:%p, %s executed. expect failed, but success.", script->fileName, - script->taos, rest); - script->linePos++; - return false; - } else { - simDebug("lineNum: %d. line: %s failed, ret:%d:%s. Expect failed, so success", line->lineNum, rest, ret & 0XFFFF, - tstrerror(ret)); - return true; - } -} diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c deleted file mode 100644 index 713e46df58..0000000000 --- a/tests/tsim/src/simMain.c +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "simInt.h" - -bool simExecSuccess = false; -bool abortExecution = false; -bool useMultiProcess = false; -bool useValgrind = false; - -void simHandleSignal(int32_t signo, void *sigInfo, void *context) { - simSystemCleanUp(); - abortExecution = true; -} - -int32_t main(int32_t argc, char *argv[]) { - char scriptFile[MAX_FILE_NAME_LEN] = "sim_main_test.sim"; - - for (int32_t i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-c") == 0 && i < argc - 1) { - tstrncpy(configDir, argv[++i], 128); - } else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) { - strcpy(scriptFile, argv[++i]); - } else if (strcmp(argv[i], "-m") == 0) { - useMultiProcess = true; - } else if (strcmp(argv[i], "-v") == 0) { - useValgrind = true; - } else { - printf("usage: %s [options] \n", argv[0]); - printf(" [-c config]: config directory, default is: %s\n", configDir); - printf(" [-f script]: script filename\n"); - return 0; - } - } - - if (!simSystemInit()) { - simError("failed to initialize the system"); - simSystemCleanUp(); - return -1; - } - - simInfo("simulator is running ..."); - taosSetSignal(SIGINT, simHandleSignal); - - SScript *script = simParseScript(scriptFile); - if (script == NULL) { - simError("parse script file:%s failed", scriptFile); - return -1; - } - - if (abortExecution) { - simError("execute abort"); - return -1; - } - - simScriptList[++simScriptPos] = script; - simExecuteScript(script); - - int32_t ret = simExecSuccess ? 0 : -1; - simInfo("execute result %d", ret); - - return ret; -} diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c deleted file mode 100644 index b9f7610be8..0000000000 --- a/tests/tsim/src/simParse.c +++ /dev/null @@ -1,1097 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -/* Thi file is to parse the simScriptPos, function file and default file, - * - * syntax - * - * expression is format like: - * $var [=|<|>|==] var - * or: - * $var = var [+|-|*|/|.] var - * - * if expression - * command - * - * if expression then - * commands - * elif expression - * commands - * elif expression - * commands - * else - * commands - * endi - * - * while expression - * commands - * continue - * break - * endw - * - * switch expression - * case 1 - * commands - * break - * case 2 - * commands - * break - * default - * commands - * ends - * - * label: - * goto label - * - */ - -#define _DEFAULT_SOURCE -#include "simParse.h" - -static SCommand *cmdHashList[MAX_NUM_CMD]; -static SCmdLine cmdLine[MAX_CMD_LINES]; -static char parseErr[MAX_ERROR_LEN]; -static char optionBuffer[MAX_OPTION_BUFFER]; -static int32_t numOfLines, optionOffset; -static SLabel label, dest; -static SBlock block; - -int32_t simHashCmd(char *token, int32_t tokenLen) { - int32_t i; - int32_t hash = 0; - - for (i = 0; i < tokenLen; ++i) hash += token[i]; - - hash = hash % MAX_NUM_CMD; - - return hash; -} - -SCommand *simCheckCmd(char *token, int32_t tokenLen) { - int32_t hash; - SCommand *node; - - hash = simHashCmd(token, tokenLen); - - node = cmdHashList[hash]; - - while (node) { - if (node->nlen == tokenLen && strncmp(node->name, token, tokenLen) == 0) { - return node; - } else { - node = node->next; - } - } - - return NULL; -} - -void simAddCmdIntoHash(SCommand *pCmd) { - int32_t hash; - SCommand *node; - - hash = simHashCmd(pCmd->name, (int32_t)strlen(pCmd->name)); - node = cmdHashList[hash]; - pCmd->next = node; - cmdHashList[hash] = pCmd; -} - -void simResetParser() { - optionOffset = 4; - numOfLines = 0; - memset(cmdLine, 0, sizeof(cmdLine)); - memset(optionBuffer, 0, sizeof(optionBuffer)); - memset(&label, 0, sizeof(label)); - memset(&block, 0, sizeof(block)); - memset(&dest, 0, sizeof(dest)); -} - -SScript *simBuildScriptObj(char *fileName) { - int32_t i, destPos; - - /* process labels */ - - cmdLine[numOfLines].cmdno = SIM_CMD_RETURN; - numOfLines++; - - for (i = 0; i < numOfLines; ++i) { - cmdLine[i].errorJump = SQL_JUMP_FALSE; - } - - for (--dest.top; dest.top >= 0; --dest.top) { - for (i = 0; i < label.top; ++i) { - if (strcmp(label.label[i], dest.label[(uint8_t)dest.top]) == 0) break; - } - - if (i == label.top) { - sprintf(parseErr, "label:%s not defined", dest.label[(uint8_t)dest.top]); - return NULL; - } - - destPos = dest.pos[(uint8_t)dest.top]; - cmdLine[destPos].jump = label.pos[i]; - if (cmdLine[destPos].cmdno == SIM_CMD_SQL) { - cmdLine[destPos].errorJump = SQL_JUMP_TRUE; - } - } - - if (block.top != 0) { - sprintf(parseErr, "mismatched block"); - return NULL; - } - - for (i = 0; i < numOfLines; ++i) { - if (cmdLine[i].jump == 0) cmdLine[i].jump = numOfLines; - } - - SScript *script = taosMemoryMalloc(sizeof(SScript)); - memset(script, 0, sizeof(SScript)); - - script->type = SIM_SCRIPT_TYPE_MAIN; - script->numOfLines = numOfLines; - tstrncpy(script->fileName, fileName, sizeof(script->fileName)); - - script->optionBuffer = taosMemoryMalloc(optionOffset); - memcpy(script->optionBuffer, optionBuffer, optionOffset); - - script->lines = taosMemoryMalloc(sizeof(SCmdLine) * numOfLines); - memcpy(script->lines, cmdLine, sizeof(SCmdLine) * numOfLines); - - return script; -} - -SScript *simParseScript(char *fileName) { - TdFilePtr pFile; - int32_t tokenLen, lineNum = 0; - char buffer[10*1024], name[128], *token, *rest; - SCommand *pCmd; - SScript *script; - - if ((fileName[0] == '.') || (fileName[0] == '/')) { - strcpy(name, fileName); - } else { - sprintf(name, "%s" TD_DIRSEP "%s", simScriptDir, fileName); - taosRealPath(name, NULL, sizeof(name)); - } - - // if ((fd = fopen(name, "r")) == NULL) { - if ((pFile = taosOpenFile(name, TD_FILE_READ | TD_FILE_STREAM)) == NULL) { - simError("failed to open file:%s", name); - return NULL; - } - - simResetParser(); - - while (!taosEOFFile(pFile)) { - if (taosGetsFile(pFile, sizeof(buffer) - 1, buffer) == -1) continue; - - lineNum++; - int32_t cmdlen = (int32_t)strlen(buffer); - if (buffer[cmdlen - 1] == '\r' || buffer[cmdlen - 1] == '\n') { - buffer[cmdlen - 1] = 0; - } - rest = buffer; - - for (int32_t i = 0; i < cmdlen; ++i) { - if (buffer[i] == '\r' || buffer[i] == '\n') { - buffer[i] = '\0'; - } - } - - again: - rest = paGetToken(rest, &token, &tokenLen); - if (tokenLen == 0) continue; - - if (token[tokenLen - 1] == ':') { - strncpy(label.label[(uint8_t)label.top], token, tokenLen - 1); - label.pos[(uint8_t)label.top] = numOfLines; - label.top++; - goto again; - } - - if (token[0] == '$') { - if (!simParseExpression(token, lineNum)) { - simError("script:%s line:%d %s", fileName, lineNum, parseErr); - return NULL; - } - continue; - } - - if ((pCmd = simCheckCmd(token, tokenLen)) == NULL) { - token[tokenLen] = 0; - simError("script:%s line:%d invalid cmd:%s", fileName, lineNum, token); - return NULL; - } - - if (!pCmd->parseCmd(rest, pCmd, lineNum)) { - simError("script:%s line:%d %s", fileName, lineNum, parseErr); - return NULL; - } - } - taosCloseFile(&pFile); - - script = simBuildScriptObj(fileName); - if (script == NULL) simError("script:%s %s", fileName, parseErr); - - return script; -} - -int32_t simCheckExpression(char *exp) { - char *op1, *op2, *op, *rest; - int32_t op1Len, op2Len, opLen; - - rest = paGetToken(exp, &op1, &op1Len); - if (op1Len == 0) { - sprintf(parseErr, "expression is required"); - return -1; - } - - rest = paGetToken(rest, &op, &opLen); - if (opLen == 0) { - sprintf(parseErr, "operator is missed"); - return -1; - } - - rest = paGetToken(rest, &op2, &op2Len); - if (op2Len == 0) { - sprintf(parseErr, "operand is missed"); - return -1; - } - - if (opLen == 1) { - if (op[0] != '=' && op[0] != '<' && op[0] != '>') { - sprintf(parseErr, "invalid operator:%s", op); - return -1; - } - - if (op[0] == '=' && op1[0] != '$') { - sprintf(parseErr, "left side of assignment must be variable"); - return -1; - } - } else if (opLen == 2) { - if (op[1] != '=' || (op[0] != '=' && op[0] != '<' && op[0] != '>' && op[0] != '!')) { - sprintf(parseErr, "left side of assignment must be variable"); - return -1; - } - } else { - sprintf(parseErr, "invalid operator:%s", op); - return -1; - } - - rest = paGetToken(rest, &op, &opLen); - - if (opLen == 0) return (int32_t)(rest - exp); - - /* if it is key word "then" */ - if (strncmp(op, "then", 4) == 0) return (int32_t)(op - exp); - - rest = paGetToken(rest, &op2, &op2Len); - if (op2Len == 0) { - sprintf(parseErr, "operand is missed"); - return -1; - } - - if (opLen > 1) { - sprintf(parseErr, "invalid operator:%s", op); - return -1; - } - - if (op[0] == '+' || op[0] == '-' || op[0] == '*' || op[0] == '/' || op[0] == '.') { - return (int32_t)(rest - exp); - } - - return -1; -} - -bool simParseExpression(char *token, int32_t lineNum) { - int32_t expLen; - - expLen = simCheckExpression(token); - if (expLen <= 0) return -1; - - cmdLine[numOfLines].cmdno = SIM_CMD_EXP; - cmdLine[numOfLines].lineNum = lineNum; - cmdLine[numOfLines].optionOffset = optionOffset; - memcpy(optionBuffer + optionOffset, token, expLen); - optionOffset += expLen + 1; - *(optionBuffer + optionOffset - 1) = 0; - - numOfLines++; - return true; -} - -bool simParseIfCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char *ret; - int32_t expLen; - - expLen = simCheckExpression(rest); - - if (expLen <= 0) return -1; - - ret = rest + expLen; - - if (strncmp(ret, "then", 4) == 0) { - block.type[(uint8_t)block.top] = BLOCK_IF; - block.pos[(uint8_t)block.top] = &cmdLine[numOfLines].jump; - block.top++; - } else { - cmdLine[numOfLines].jump = numOfLines + 2; - } - - cmdLine[numOfLines].optionOffset = optionOffset; - memcpy(optionBuffer + optionOffset, rest, expLen); - optionOffset += expLen + 1; - *(optionBuffer + optionOffset - 1) = 0; - cmdLine[numOfLines].cmdno = SIM_CMD_TEST; - cmdLine[numOfLines].lineNum = lineNum; - - numOfLines++; - return true; -} - -bool simParseElifCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - int32_t expLen; - - expLen = simCheckExpression(rest); - - if (expLen <= 0) return -1; - - if (block.top < 1) { - sprintf(parseErr, "no matching if"); - return false; - } - - if (block.type[block.top - 1] != BLOCK_IF) { - sprintf(parseErr, "no matched if block"); - return false; - } - - cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; - block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump); - block.numJump[block.top - 1]++; - - numOfLines++; - - *(block.pos[block.top - 1]) = numOfLines; - block.pos[block.top - 1] = &cmdLine[numOfLines].jump; - - cmdLine[numOfLines].optionOffset = optionOffset; - memcpy(optionBuffer + optionOffset, rest, expLen); - optionOffset += expLen + 1; - *(optionBuffer + optionOffset - 1) = 0; - cmdLine[numOfLines].cmdno = SIM_CMD_TEST; - cmdLine[numOfLines].lineNum = lineNum; - - numOfLines++; - return true; -} - -bool simParseElseCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - if (block.top < 1) { - sprintf(parseErr, "no matching if"); - return false; - } - - if (block.type[block.top - 1] != BLOCK_IF) { - sprintf(parseErr, "no matched if block"); - return false; - } - - cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; - block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump); - block.numJump[block.top - 1]++; - - numOfLines++; - - *(block.pos[block.top - 1]) = numOfLines; - block.pos[block.top - 1] = NULL; - - return true; -} - -bool simParseEndiCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - int32_t i; - - if (block.top < 1) { - sprintf(parseErr, "no matching if"); - return false; - } - - if (block.type[block.top - 1] != BLOCK_IF) { - sprintf(parseErr, "no matched if block"); - return false; - } - - if (block.pos[block.top - 1]) *(block.pos[block.top - 1]) = numOfLines; - - for (i = 0; i < block.numJump[block.top - 1]; ++i) { - *(block.jump[block.top - 1][i]) = numOfLines; - } - - block.numJump[block.top - 1] = 0; - block.top--; - - return true; -} - -bool simParseWhileCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - int32_t expLen; - - expLen = simCheckExpression(rest); - - if (expLen <= 0) return false; - - block.type[(uint8_t)block.top] = BLOCK_WHILE; - block.pos[(uint8_t)block.top] = &(cmdLine[numOfLines].jump); - block.back[(uint8_t)block.top] = numOfLines; - block.top++; - - cmdLine[numOfLines].optionOffset = optionOffset; - memcpy(optionBuffer + optionOffset, rest, expLen); - optionOffset += expLen + 1; - *(optionBuffer + optionOffset - 1) = 0; - cmdLine[numOfLines].cmdno = SIM_CMD_TEST; - cmdLine[numOfLines].lineNum = lineNum; - - numOfLines++; - return true; -} - -bool simParseEndwCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - int32_t i; - - if (block.top < 1) { - sprintf(parseErr, "no matching while"); - return false; - } - - if (block.type[block.top - 1] != BLOCK_WHILE) { - sprintf(parseErr, "no matched while block"); - return false; - } - - cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; - cmdLine[numOfLines].jump = block.back[block.top - 1]; - cmdLine[numOfLines].lineNum = lineNum; - numOfLines++; - - *(block.pos[block.top - 1]) = numOfLines; - - for (i = 0; i < block.numJump[block.top - 1]; ++i) { - *(block.jump[block.top - 1][i]) = numOfLines; - } - - block.top--; - - return true; -} - -bool simParseSwitchCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char *token; - int32_t tokenLen; - - rest = paGetToken(rest, &token, &tokenLen); - if (tokenLen == 0) { - sprintf(parseErr, "switch should be followed by variable"); - return false; - } - - if (token[0] != '$') { - sprintf(parseErr, "switch must be followed by variable"); - return false; - } - - memcpy(block.sexp[(uint8_t)block.top], token, tokenLen); - block.sexpLen[(uint8_t)block.top] = tokenLen; - block.type[(uint8_t)block.top] = BLOCK_SWITCH; - block.top++; - - return true; -} - -bool simParseCaseCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char *token; - int32_t tokenLen; - - rest = paGetToken(rest, &token, &tokenLen); - if (tokenLen == 0) { - sprintf(parseErr, "case should be followed by value"); - return false; - } - - if (block.top < 1) { - sprintf(parseErr, "no matching switch"); - return false; - } - - if (block.type[block.top - 1] != BLOCK_SWITCH) { - sprintf(parseErr, "case not matched"); - return false; - } - - if (block.pos[block.top - 1] != NULL) { - *(block.pos[block.top - 1]) = numOfLines; - } - - block.pos[block.top - 1] = &(cmdLine[numOfLines].jump); - - cmdLine[numOfLines].cmdno = SIM_CMD_TEST; - cmdLine[numOfLines].lineNum = lineNum; - cmdLine[numOfLines].optionOffset = optionOffset; - memcpy(optionBuffer + optionOffset, block.sexp[block.top - 1], block.sexpLen[block.top - 1]); - optionOffset += block.sexpLen[block.top - 1]; - *(optionBuffer + optionOffset++) = ' '; - *(optionBuffer + optionOffset++) = '='; - *(optionBuffer + optionOffset++) = '='; - *(optionBuffer + optionOffset++) = ' '; - memcpy(optionBuffer + optionOffset, token, tokenLen); - optionOffset += tokenLen + 1; - *(optionBuffer + optionOffset - 1) = 0; - - numOfLines++; - return true; -} - -bool simParseBreakCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - if (block.top < 1) { - sprintf(parseErr, "no blcok exists"); - return false; - } - - if (block.type[block.top - 1] != BLOCK_SWITCH && block.type[block.top - 1] != BLOCK_WHILE) { - sprintf(parseErr, "not in switch or while block"); - return false; - } - - block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump); - block.numJump[block.top - 1]++; - - cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; - cmdLine[numOfLines].lineNum = lineNum; - - numOfLines++; - return true; -} - -bool simParseDefaultCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - if (block.top < 1) { - sprintf(parseErr, "no matching switch"); - return false; - } - - if (block.type[block.top - 1] != BLOCK_SWITCH) { - sprintf(parseErr, "default should be matched with switch"); - return false; - } - - if (block.pos[block.top - 1] != NULL) { - *(block.pos[block.top - 1]) = numOfLines; - } - - return true; -} - -bool simParseEndsCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - int32_t i; - - if (block.top < 1) { - sprintf(parseErr, "no matching switch"); - return false; - } - - if (block.type[block.top - 1] != BLOCK_SWITCH) { - sprintf(parseErr, "ends should be matched with switch"); - return false; - } - - for (i = 0; i < block.numJump[block.top - 1]; ++i) { - *(block.jump[block.top - 1][i]) = numOfLines; - } - - block.numJump[block.top - 1] = 0; - block.top--; - - return true; -} - -bool simParseContinueCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - if (block.top < 1) { - sprintf(parseErr, "no matching while"); - return false; - } - - if (block.type[block.top - 1] != BLOCK_WHILE) { - sprintf(parseErr, "continue should be matched with while cmd"); - return false; - } - - cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; - cmdLine[numOfLines].lineNum = lineNum; - cmdLine[numOfLines].jump = block.back[block.top - 1]; - - numOfLines++; - return true; -} - -bool simParsePrintCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - int32_t expLen; - - rest++; - cmdLine[numOfLines].cmdno = SIM_CMD_PRINT; - cmdLine[numOfLines].lineNum = lineNum; - cmdLine[numOfLines].optionOffset = optionOffset; - expLen = (int32_t)strlen(rest); - memcpy(optionBuffer + optionOffset, rest, expLen); - optionOffset += expLen + 1; - *(optionBuffer + optionOffset - 1) = 0; - - numOfLines++; - return true; -} - -void simCheckSqlOption(char *rest) { - int32_t valueLen; - char *value, *xpos; - - xpos = strstr(rest, " -x"); // need a blank - if (xpos) { - paGetToken(xpos + 3, &value, &valueLen); - if (valueLen != 0) { - memcpy(dest.label[(uint8_t)dest.top], value, valueLen); - dest.label[(uint8_t)dest.top][valueLen] = 0; - dest.pos[(uint8_t)dest.top] = numOfLines; - dest.top++; - - *xpos = 0; - } - } -} - -bool simParseSqlCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - int32_t expLen; - - rest++; - simCheckSqlOption(rest); - cmdLine[numOfLines].cmdno = SIM_CMD_SQL; - cmdLine[numOfLines].lineNum = lineNum; - cmdLine[numOfLines].optionOffset = optionOffset; - expLen = (int32_t)strlen(rest); - memcpy(optionBuffer + optionOffset, rest, expLen); - optionOffset += expLen + 1; - *(optionBuffer + optionOffset - 1) = 0; - - numOfLines++; - return true; -} - -bool simParseSqlErrorCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - int32_t expLen; - - rest++; - cmdLine[numOfLines].cmdno = SIM_CMD_SQL_ERROR; - cmdLine[numOfLines].lineNum = lineNum; - cmdLine[numOfLines].optionOffset = optionOffset; - expLen = (int32_t)strlen(rest); - memcpy(optionBuffer + optionOffset, rest, expLen); - optionOffset += expLen + 1; - *(optionBuffer + optionOffset - 1) = 0; - - numOfLines++; - return true; -} - -bool simParseSqlSlowCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - simParseSqlCmd(rest, pCmd, lineNum); - cmdLine[numOfLines - 1].cmdno = SIM_CMD_SQL_SLOW; - return true; -} - -bool simParseRestfulCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - simParseSqlCmd(rest, pCmd, lineNum); - cmdLine[numOfLines - 1].cmdno = SIM_CMD_RESTFUL; - return true; -} - -bool simParseSystemCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - int32_t expLen; - - rest++; - cmdLine[numOfLines].cmdno = SIM_CMD_SYSTEM; - cmdLine[numOfLines].lineNum = lineNum; - cmdLine[numOfLines].optionOffset = optionOffset; - expLen = (int32_t)strlen(rest); - memcpy(optionBuffer + optionOffset, rest, expLen); - optionOffset += expLen + 1; - *(optionBuffer + optionOffset - 1) = 0; - - numOfLines++; - return true; -} - -bool simParseSystemContentCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - simParseSystemCmd(rest, pCmd, lineNum); - cmdLine[numOfLines - 1].cmdno = SIM_CMD_SYSTEM_CONTENT; - return true; -} - -bool simParseSleepCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char *token; - int32_t tokenLen; - - cmdLine[numOfLines].cmdno = SIM_CMD_SLEEP; - cmdLine[numOfLines].lineNum = lineNum; - - paGetToken(rest, &token, &tokenLen); - if (tokenLen > 0) { - cmdLine[numOfLines].optionOffset = optionOffset; - memcpy(optionBuffer + optionOffset, token, tokenLen); - optionOffset += tokenLen + 1; - *(optionBuffer + optionOffset - 1) = 0; - } - - numOfLines++; - return true; -} - -bool simParseReturnCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char *token; - int32_t tokenLen; - - cmdLine[numOfLines].cmdno = SIM_CMD_RETURN; - cmdLine[numOfLines].lineNum = lineNum; - - paGetToken(rest, &token, &tokenLen); - if (tokenLen > 0) { - cmdLine[numOfLines].optionOffset = optionOffset; - memcpy(optionBuffer + optionOffset, token, tokenLen); - optionOffset += tokenLen + 1; - *(optionBuffer + optionOffset - 1) = 0; - } - - numOfLines++; - return true; -} - -bool simParseGotoCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char *token; - int32_t tokenLen; - - rest = paGetToken(rest, &token, &tokenLen); - - if (tokenLen == 0) { - sprintf(parseErr, "label should be followed by goto cmd"); - return false; - } - - memcpy(dest.label[(uint8_t)dest.top], token, tokenLen); - dest.label[(uint8_t)dest.top][tokenLen] = 0; - dest.pos[(uint8_t)dest.top] = numOfLines; - dest.top++; - - cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; - cmdLine[numOfLines].lineNum = lineNum; - - numOfLines++; - return true; -} - -bool simParseRunCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char *token; - int32_t tokenLen; - - rest = paGetToken(rest, &token, &tokenLen); - - if (tokenLen == 0) { - sprintf(parseErr, "file name should be followed by run cmd"); - return false; - } - - cmdLine[numOfLines].cmdno = SIM_CMD_RUN; - cmdLine[numOfLines].lineNum = lineNum; - cmdLine[numOfLines].optionOffset = optionOffset; - memcpy(optionBuffer + optionOffset, token, tokenLen); - optionOffset += tokenLen + 1; - *(optionBuffer + optionOffset - 1) = 0; - - numOfLines++; - return true; -} - -bool simParseRunBackCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - simParseRunCmd(rest, pCmd, lineNum); - cmdLine[numOfLines - 1].cmdno = SIM_CMD_RUN_BACK; - return true; -} - -bool simParseLineInsertCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - int32_t expLen; - - rest++; - cmdLine[numOfLines].cmdno = SIM_CMD_LINE_INSERT; - cmdLine[numOfLines].lineNum = lineNum; - cmdLine[numOfLines].optionOffset = optionOffset; - expLen = (int32_t)strlen(rest); - memcpy(optionBuffer + optionOffset, rest, expLen); - optionOffset += expLen + 1; - *(optionBuffer + optionOffset - 1) = 0; - - numOfLines++; - return true; -} - -bool simParseLineInsertErrorCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - int32_t expLen; - - rest++; - cmdLine[numOfLines].cmdno = SIM_CMD_LINE_INSERT; - cmdLine[numOfLines].lineNum = lineNum; - cmdLine[numOfLines].optionOffset = optionOffset; - expLen = (int32_t)strlen(rest); - memcpy(optionBuffer + optionOffset, rest, expLen); - optionOffset += expLen + 1; - *(optionBuffer + optionOffset - 1) = 0; - - numOfLines++; - return true; -} - -void simInitsimCmdList() { - int32_t cmdno; - memset(simCmdList, 0, SIM_CMD_END * sizeof(SCommand)); - - /* internal command */ - cmdno = SIM_CMD_EXP; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "exp"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = NULL; - simCmdList[cmdno].executeCmd = simExecuteExpCmd; - - cmdno = SIM_CMD_IF; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "if"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseIfCmd; - simCmdList[cmdno].executeCmd = NULL; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_ELIF; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "elif"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseElifCmd; - simCmdList[cmdno].executeCmd = NULL; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_ELSE; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "else"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseElseCmd; - simCmdList[cmdno].executeCmd = NULL; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_ENDI; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "endi"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseEndiCmd; - simCmdList[cmdno].executeCmd = NULL; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_WHILE; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "while"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseWhileCmd; - simCmdList[cmdno].executeCmd = NULL; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_ENDW; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "endw"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseEndwCmd; - simCmdList[cmdno].executeCmd = NULL; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_SWITCH; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "switch"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseSwitchCmd; - simCmdList[cmdno].executeCmd = NULL; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_CASE; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "case"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseCaseCmd; - simCmdList[cmdno].executeCmd = NULL; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_DEFAULT; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "default"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseDefaultCmd; - simCmdList[cmdno].executeCmd = NULL; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_BREAK; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "break"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseBreakCmd; - simCmdList[cmdno].executeCmd = NULL; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_CONTINUE; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "continue"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseContinueCmd; - simCmdList[cmdno].executeCmd = NULL; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_ENDS; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "ends"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseEndsCmd; - simCmdList[cmdno].executeCmd = NULL; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_SLEEP; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "sleep"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseSleepCmd; - simCmdList[cmdno].executeCmd = simExecuteSleepCmd; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_GOTO; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "goto"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseGotoCmd; - simCmdList[cmdno].executeCmd = simExecuteGotoCmd; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_RUN; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "run"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseRunCmd; - simCmdList[cmdno].executeCmd = simExecuteRunCmd; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_RUN_BACK; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "run_back"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseRunBackCmd; - simCmdList[cmdno].executeCmd = simExecuteRunBackCmd; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_SYSTEM; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "system"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseSystemCmd; - simCmdList[cmdno].executeCmd = simExecuteSystemCmd; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_SYSTEM_CONTENT; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "system_content"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseSystemContentCmd; - simCmdList[cmdno].executeCmd = simExecuteSystemContentCmd; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_PRINT; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "print"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParsePrintCmd; - simCmdList[cmdno].executeCmd = simExecutePrintCmd; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_SQL; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "sql"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseSqlCmd; - simCmdList[cmdno].executeCmd = simExecuteSqlCmd; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_SQL_ERROR; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "sql_error"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseSqlErrorCmd; - simCmdList[cmdno].executeCmd = simExecuteSqlErrorCmd; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_SQL_SLOW; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "sql_slow"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseSqlSlowCmd; - simCmdList[cmdno].executeCmd = simExecuteSqlSlowCmd; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_RESTFUL; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "restful"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseRestfulCmd; - simCmdList[cmdno].executeCmd = simExecuteRestfulCmd; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - /* test is only an internal command */ - cmdno = SIM_CMD_TEST; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "test"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = NULL; - simCmdList[cmdno].executeCmd = simExecuteTestCmd; - - cmdno = SIM_CMD_RETURN; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "return"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseReturnCmd; - simCmdList[cmdno].executeCmd = simExecuteReturnCmd; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_LINE_INSERT; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "line_insert"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseLineInsertCmd; - simCmdList[cmdno].executeCmd = simExecuteLineInsertCmd; - simAddCmdIntoHash(&(simCmdList[cmdno])); - - cmdno = SIM_CMD_LINE_INSERT_ERROR; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "line_insert_error"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseLineInsertErrorCmd; - simCmdList[cmdno].executeCmd = simExecuteLineInsertErrorCmd; - simAddCmdIntoHash(&(simCmdList[cmdno])); -} diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c deleted file mode 100644 index f2fefb903d..0000000000 --- a/tests/tsim/src/simSystem.c +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "simInt.h" -#include "tconfig.h" - -SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; -SCommand simCmdList[SIM_CMD_END]; -int32_t simScriptPos = -1; -int32_t simScriptSucced = 0; -int32_t simDebugFlag = 143; -void simCloseTaosdConnect(SScript *script); -char simScriptDir[PATH_MAX] = {0}; - -extern bool simExecSuccess; - -int32_t simInitCfg() { - taosCreateLog("simlog", 1, configDir, NULL, NULL, NULL, NULL, 1); - taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1); - - SConfig *pCfg = taosGetCfg(); - simDebugFlag = cfgGetItem(pCfg, "simDebugFlag")->i32; - tstrncpy(simScriptDir, cfgGetItem(pCfg, "scriptDir")->str, PATH_MAX); - return 0; -} - -bool simSystemInit() { - simInitCfg(); - simInitsimCmdList(); - memset(simScriptList, 0, sizeof(SScript *) * MAX_MAIN_SCRIPT_NUM); - return true; -} - -void simSystemCleanUp() {} - -void simFreeScript(SScript *script) { - if (script->type == SIM_SCRIPT_TYPE_MAIN) { - simInfo("script:%s, background script num:%d, stop them", script->fileName, script->bgScriptLen); - - for (int32_t i = 0; i < script->bgScriptLen; ++i) { - SScript *bgScript = script->bgScripts[i]; - simDebug("script:%s, is background script, set stop flag", bgScript->fileName); - bgScript->killed = true; - if (taosCheckPthreadValid(bgScript->bgPid)) { - taosThreadJoin(bgScript->bgPid, NULL); - taosThreadClear(&bgScript->bgPid); - } - - simDebug("script:%s, background thread joined", bgScript->fileName); - taos_close(bgScript->taos); - taosMemoryFreeClear(bgScript->lines); - taosMemoryFreeClear(bgScript->optionBuffer); - taosMemoryFreeClear(bgScript); - } - - simDebug("script:%s, is cleaned", script->fileName); - taos_close(script->taos); - taosMemoryFreeClear(script->lines); - taosMemoryFreeClear(script->optionBuffer); - taosMemoryFreeClear(script); - } -} - -SScript *simProcessCallOver(SScript *script) { - if (script->type == SIM_SCRIPT_TYPE_MAIN) { - simDebug("script:%s, is main script, set stop flag", script->fileName); - if (script->killed) { - simExecSuccess = false; - simInfo("script:" FAILED_PREFIX "%s" FAILED_POSTFIX ", " FAILED_PREFIX "failed" FAILED_POSTFIX ", error:%s", - script->fileName, script->error); - } else { - simExecSuccess = true; - simInfo("script:" SUCCESS_PREFIX "%s" SUCCESS_POSTFIX ", " SUCCESS_PREFIX "success" SUCCESS_POSTFIX, - script->fileName); - } - - simCloseTaosdConnect(script); - simScriptSucced++; - simScriptPos--; - simFreeScript(script); - - if (simScriptPos == -1 && simExecSuccess) { - simInfo("----------------------------------------------------------------------"); - simInfo("Simulation Test Done, " SUCCESS_PREFIX "%d" SUCCESS_POSTFIX " Passed:\n", simScriptSucced); - return NULL; - } - - if (simScriptPos == -1) return NULL; - if (!simExecSuccess) return NULL; - - return simScriptList[simScriptPos]; - } else { - simDebug("script:%s, is stopped", script->fileName); - simFreeScript(script); - return NULL; - } -} - -void *simExecuteScript(void *inputScript) { - SScript *script = (SScript *)inputScript; - - while (1) { - if (script->type == SIM_SCRIPT_TYPE_MAIN) { - script = simScriptList[simScriptPos]; - } - - if (abortExecution) { - script->killed = true; - } - - if (script->killed || script->linePos >= script->numOfLines) { - script = simProcessCallOver(script); - if (script == NULL) { - simDebug("sim test abort now!"); - break; - } - } else { - SCmdLine *line = &script->lines[script->linePos]; - char *option = script->optionBuffer + line->optionOffset; - simDebug("script:%s, line:%d with option \"%s\"", script->fileName, line->lineNum, option); - - SCommand *cmd = &simCmdList[line->cmdno]; - int32_t ret = (*(cmd->executeCmd))(script, option); - if (!ret) { - script->killed = true; - } - } - } - - simInfo("thread is stopped"); - return NULL; -} From b159d3cdf54a0e355a5235233de369efab5365b4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Sep 2022 10:05:57 +0800 Subject: [PATCH 13/16] fix(query): fix syntax error on windows platform --- include/util/trbtree.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/trbtree.h b/include/util/trbtree.h index 50e2663648..0ca7e3ba95 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -51,7 +51,7 @@ struct SRBTreeNode { SRBTreeNode *parent; SRBTreeNode *left; SRBTreeNode *right; - uint8_t payload[0]; + uint8_t payload[]; }; struct SRBTree { From 4ace5bef9c60830e5564a675c38fdce5c3010276 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Sep 2022 11:08:36 +0800 Subject: [PATCH 14/16] fix(query): fix syntax error. --- include/util/trbtree.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/trbtree.h b/include/util/trbtree.h index 0ca7e3ba95..50e2663648 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -51,7 +51,7 @@ struct SRBTreeNode { SRBTreeNode *parent; SRBTreeNode *left; SRBTreeNode *right; - uint8_t payload[]; + uint8_t payload[0]; }; struct SRBTree { From bbbbebab893ea78233dc8ddbafac36aa730d3b44 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 3 Sep 2022 13:37:48 +0800 Subject: [PATCH 15/16] fix: windows compile --- include/util/trbtree.h | 3 ++- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 11 ++++++----- source/util/src/trbtree.c | 10 +++++----- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/include/util/trbtree.h b/include/util/trbtree.h index 50e2663648..f6d37e3d75 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -51,9 +51,10 @@ struct SRBTreeNode { SRBTreeNode *parent; SRBTreeNode *left; SRBTreeNode *right; - uint8_t payload[0]; }; +#define RBTREE_NODE_PAYLOAD(N) ((const void *)&(N)[1]) + struct SRBTree { tRBTreeCmprFn cmprFn; int64_t n; diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index ce36d74467..25f57e21fc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -273,7 +273,8 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { } } -int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange) { +int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t uid, + STimeWindow *pTimeWindow, SVersionRange *pVerRange) { pMTree->backward = backward; pMTree->pIter = NULL; pMTree->pIterList = taosArrayInit(4, POINTER_BYTES); @@ -284,8 +285,8 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFRead tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); int32_t code = TSDB_CODE_OUT_OF_MEMORY; - struct SLDataIter* pIterList[TSDB_DEFAULT_LAST_FILE] = {0}; - for(int32_t i = 0; i < pFReader->pSet->nSstF; ++i) { // open all last file + struct SLDataIter *pIterList[TSDB_DEFAULT_LAST_FILE] = {0}; + for (int32_t i = 0; i < pFReader->pSet->nSstF; ++i) { // open all last file code = tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, uid, pTimeWindow, pVerRange); if (code != TSDB_CODE_SUCCESS) { goto _end; @@ -302,7 +303,7 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFRead return code; - _end: +_end: tMergeTreeClose(pMTree); return code; } @@ -322,7 +323,7 @@ bool tMergeTreeNext(SMergeTree *pMTree) { // compare with min in RB Tree pIter = (SLDataIter *)tRBTreeMin(&pMTree->rbt); if (pMTree->pIter && pIter) { - int32_t c = pMTree->rbt.cmprFn(pMTree->pIter->node.payload, &pIter->node.payload); + int32_t c = pMTree->rbt.cmprFn(RBTREE_NODE_PAYLOAD(&pMTree->pIter->node), RBTREE_NODE_PAYLOAD(&pIter->node)); if (c > 0) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter); pMTree->pIter = NULL; diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index 2d81a6c5b4..65f1bac60a 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -219,7 +219,7 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { while (temp != pTree->NIL) { y = temp; - int32_t c = pTree->cmprFn(z->payload, temp->payload); + int32_t c = pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(temp)); if (c < 0) { temp = temp->left; } else if (c > 0) { @@ -232,7 +232,7 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { if (y == pTree->NIL) { pTree->root = z; - } else if (pTree->cmprFn(z->payload, y->payload) < 0) { + } else if (pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(y)) < 0) { y->left = z; } else { y->right = z; @@ -245,10 +245,10 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { tRBTreePutFix(pTree, z); // update min/max node - if (pTree->min == pTree->NIL || pTree->cmprFn(pTree->min->payload, z->payload) > 0) { + if (pTree->min == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->min), RBTREE_NODE_PAYLOAD(z)) > 0) { pTree->min = z; } - if (pTree->max == pTree->NIL || pTree->cmprFn(pTree->max->payload, z->payload) < 0) { + if (pTree->max == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->max), RBTREE_NODE_PAYLOAD(z)) < 0) { pTree->max = z; } pTree->n++; @@ -313,7 +313,7 @@ SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { SRBTreeNode *pNode = pTree->root; while (pNode != pTree->NIL) { - int32_t c = pTree->cmprFn(pKey, pNode->payload); + int32_t c = pTree->cmprFn(pKey, RBTREE_NODE_PAYLOAD(pNode)); if (c < 0) { pNode = pNode->left; From 82f0ba3655f29f07372b983e22071f45b3475dfc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 3 Sep 2022 14:18:29 +0800 Subject: [PATCH 16/16] fix: windows compile error --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 25f57e21fc..b388e5a0fe 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -252,8 +252,8 @@ SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; } // SMergeTree ================================================= static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { - SLDataIter *pIter1 = (SLDataIter *)(p1 - sizeof(SRBTreeNode)); - SLDataIter *pIter2 = (SLDataIter *)(p2 - sizeof(SRBTreeNode)); + SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - sizeof(SRBTreeNode)); + SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - sizeof(SRBTreeNode)); TSDBKEY key1 = TSDBROW_KEY(&pIter1->rInfo.row); TSDBKEY key2 = TSDBROW_KEY(&pIter2->rInfo.row);