From 29ba8ffb2d0da91cb1632cff4f7cdebc91d4404a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 5 Oct 2024 16:12:59 +0800 Subject: [PATCH 01/34] refactor: init the local variable value. --- source/libs/stream/src/streamTask.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 71a2ed3e4a..53b262dff5 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -884,7 +884,7 @@ int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskI } int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask) { - int32_t code; + int32_t code = 0; int32_t tlen = 0; int32_t vgId = pTask->pMeta->vgId; const char* id = pTask->id.idStr; From 6170bb5c1cd550536c86c2bf3279dcb78a10ef58 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Oct 2024 09:39:32 +0800 Subject: [PATCH 02/34] enh(stream): check the existence for snode. --- source/dnode/mnode/impl/inc/mndStream.h | 1 + source/dnode/mnode/impl/src/mndStream.c | 19 ++++++++++++---- source/dnode/mnode/impl/src/mndStreamUtil.c | 24 +++++++++++++++++++++ source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- source/libs/stream/src/streamTimer.c | 2 +- 5 files changed, 42 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index b97eaf31d1..ebc2e50065 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -133,6 +133,7 @@ int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgr int32_t mndGetStreamObj(SMnode *pMnode, int64_t streamId, SStreamObj** pStream); bool mndStreamNodeIsUpdated(SMnode *pMnode); +bool mndCheckForSnode(SMnode *pMnode, SDbObj *pSrcDb); int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId); int32_t mndProcessStreamHb(SRpcMsg *pReq); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 3ec99f6e44..d486570300 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -801,12 +801,23 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { } if (createReq.sql != NULL) { - sqlLen = strlen(createReq.sql); - sql = taosMemoryMalloc(sqlLen + 1); + sql = taosStrdup(createReq.sql); TSDB_CHECK_NULL(sql, code, lino, _OVER, terrno); + } - memset(sql, 0, sqlLen + 1); - memcpy(sql, createReq.sql, sqlLen); + SDbObj *pSourceDb = mndAcquireDb(pMnode, createReq.sourceDB); + if (pSourceDb == NULL) { + code = terrno; + mInfo("stream:%s failed to create, acquire source db %s failed, code:%s", createReq.name, createReq.sourceDB, + tstrerror(code)); + goto _OVER; + } + + bool snodeCheckSucc = mndCheckForSnode(pMnode, pSourceDb); + mndReleaseDb(pMnode, pSourceDb); + if (!snodeCheckSucc) { + code = TSDB_CODE_SNODE_NOT_DEPLOYED; + goto _OVER; } // build stream obj from request diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 6e48c58b30..206422b795 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -1497,6 +1497,30 @@ bool mndStreamNodeIsUpdated(SMnode *pMnode) { return updated; } +bool mndCheckForSnode(SMnode *pMnode, SDbObj *pSrcDb) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SSnodeObj *pObj = NULL; + + if (pSrcDb->cfg.replications == 1) { + return true; + } else { + while (1) { + pIter = sdbFetch(pSdb, SDB_SNODE, pIter, (void **)&pObj); + if (pIter == NULL) { + break; + } + + sdbRelease(pSdb, pObj); + sdbCancelFetch(pSdb, pIter); + return true; + } + + mError("snode not existed when trying to create stream in db with multiple replica"); + return false; + } +} + uint32_t seed = 0; static SRpcMsg createRpcMsg(STransAction* pAction, int64_t traceId, int64_t signature) { SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen, .info.ahandle = (void *)signature}; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 9dbf16cb48..ab8eede0dd 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -629,7 +629,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg } break; case TDMT_STREAM_TASK_DEPLOY: { - int32_t code = tqProcessTaskDeployReq(pVnode->pTq, ver, pReq, len); + code = tqProcessTaskDeployReq(pVnode->pTq, ver, pReq, len); if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _err; diff --git a/source/libs/stream/src/streamTimer.c b/source/libs/stream/src/streamTimer.c index 8b77fe7cb1..0da9acfd1d 100644 --- a/source/libs/stream/src/streamTimer.c +++ b/source/libs/stream/src/streamTimer.c @@ -56,7 +56,7 @@ void streamTmrStart(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* pParam, void* } } - stDebug("vgId:%d start %s tmr succ", vgId, pMsg); + stTrace("vgId:%d start %s tmr succ", vgId, pMsg); } void streamTmrStop(tmr_h tmrId) { From 39495ec935a617aad3d7358dc6dc51cf41abe49a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 9 Oct 2024 10:07:08 +0800 Subject: [PATCH 03/34] refactor: do some internal refactor. --- source/dnode/mnode/impl/inc/mndStream.h | 2 +- source/dnode/mnode/impl/src/mndStream.c | 5 ++--- source/dnode/mnode/impl/src/mndStreamUtil.c | 8 ++++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index ebc2e50065..c9155f536c 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -133,7 +133,7 @@ int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgr int32_t mndGetStreamObj(SMnode *pMnode, int64_t streamId, SStreamObj** pStream); bool mndStreamNodeIsUpdated(SMnode *pMnode); -bool mndCheckForSnode(SMnode *pMnode, SDbObj *pSrcDb); +int32_t mndCheckForSnode(SMnode *pMnode, SDbObj *pSrcDb); int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId); int32_t mndProcessStreamHb(SRpcMsg *pReq); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index d486570300..9c22543265 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -813,10 +813,9 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - bool snodeCheckSucc = mndCheckForSnode(pMnode, pSourceDb); + code = mndCheckForSnode(pMnode, pSourceDb); mndReleaseDb(pMnode, pSourceDb); - if (!snodeCheckSucc) { - code = TSDB_CODE_SNODE_NOT_DEPLOYED; + if (code != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 206422b795..423d9df4b6 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -1497,13 +1497,13 @@ bool mndStreamNodeIsUpdated(SMnode *pMnode) { return updated; } -bool mndCheckForSnode(SMnode *pMnode, SDbObj *pSrcDb) { +int32_t mndCheckForSnode(SMnode *pMnode, SDbObj *pSrcDb) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; SSnodeObj *pObj = NULL; if (pSrcDb->cfg.replications == 1) { - return true; + return TSDB_CODE_SUCCESS; } else { while (1) { pIter = sdbFetch(pSdb, SDB_SNODE, pIter, (void **)&pObj); @@ -1513,11 +1513,11 @@ bool mndCheckForSnode(SMnode *pMnode, SDbObj *pSrcDb) { sdbRelease(pSdb, pObj); sdbCancelFetch(pSdb, pIter); - return true; + return TSDB_CODE_SUCCESS; } mError("snode not existed when trying to create stream in db with multiple replica"); - return false; + return TSDB_CODE_SNODE_NOT_DEPLOYED; } } From dca556a7e8aa542d60e9d96818b24a414550febd Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Mon, 30 Sep 2024 16:52:02 +0800 Subject: [PATCH 04/34] enh: delay allocation of array in LRU-insert --- source/util/src/tlrucache.c | 39 ++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index 69832cd46c..92dcf015a6 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -372,23 +372,34 @@ static void taosLRUCacheShardCleanup(SLRUCacheShard *shard) { static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry *e, LRUHandle **handle, bool freeOnFail) { LRUStatus status = TAOS_LRU_STATUS_OK; - SArray *lastReferenceList = taosArrayInit(16, POINTER_BYTES); - if (!lastReferenceList) { - taosLRUEntryFree(e); - return TAOS_LRU_STATUS_FAIL; + SLRUEntry *toFree = NULL; + SArray *lastReferenceList = NULL; + if (shard->usage + e->totalCharge > shard->capacity) { + lastReferenceList = taosArrayInit(16, POINTER_BYTES); + if (!lastReferenceList) { + taosLRUEntryFree(e); + return TAOS_LRU_STATUS_FAIL; + } } (void)taosThreadMutexLock(&shard->mutex); - taosLRUCacheShardEvictLRU(shard, e->totalCharge, lastReferenceList); + if (shard->usage + e->totalCharge > shard->capacity && shard->lru.next != &shard->lru) { + if (!lastReferenceList) { + lastReferenceList = taosArrayInit(16, POINTER_BYTES); + if (!lastReferenceList) { + taosLRUEntryFree(e); + (void)taosThreadMutexUnlock(&shard->mutex); + return TAOS_LRU_STATUS_FAIL; + } + } + taosLRUCacheShardEvictLRU(shard, e->totalCharge, lastReferenceList); + } if (shard->usage + e->totalCharge > shard->capacity && (shard->strictCapacity || handle == NULL)) { TAOS_LRU_ENTRY_SET_IN_CACHE(e, false); if (handle == NULL) { - if (!taosArrayPush(lastReferenceList, &e)) { - taosLRUEntryFree(e); - goto _exit; - } + toFree = e; } else { if (freeOnFail) { taosLRUEntryFree(e); @@ -413,11 +424,7 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry * taosLRUCacheShardLRURemove(shard, old); shard->usage -= old->totalCharge; - if (!taosArrayPush(lastReferenceList, &old)) { - taosLRUEntryFree(e); - taosLRUEntryFree(old); - goto _exit; - } + toFree = old; } } if (handle == NULL) { @@ -434,6 +441,10 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry * _exit: (void)taosThreadMutexUnlock(&shard->mutex); + if (toFree) { + taosLRUEntryFree(toFree); + } + for (int i = 0; i < taosArrayGetSize(lastReferenceList); ++i) { SLRUEntry *entry = taosArrayGetP(lastReferenceList, i); From eaae6d4bfda93383a1b7c30d64e987eda79ee93f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 21 Oct 2024 17:30:12 +0800 Subject: [PATCH 05/34] enh:[TD-32166]refactor code in sml --- source/client/inc/clientSml.h | 24 +- source/client/src/clientSml.c | 1218 ++++++++----------------- source/client/src/clientSmlJson.c | 68 +- source/client/src/clientSmlLine.c | 28 +- source/client/src/clientSmlTelnet.c | 38 +- source/client/test/smlTest.cpp | 98 ++ source/libs/parser/src/parInsertSml.c | 23 +- 7 files changed, 592 insertions(+), 905 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index a6aca2fddf..3098b124b7 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -92,6 +92,22 @@ extern "C" { } \ } +#define SML_CHECK_CODE(CMD) \ + do { \ + code = (CMD); \ + if (TSDB_CODE_SUCCESS != code) { \ + goto END; \ + } \ + } while (0) + +#define SML_CHECK_NULL(CMD) \ + do { \ + if (NULL == (CMD)) { \ + code = terrno; \ + goto END; \ + } \ + } while (0) + typedef enum { SCHEMA_ACTION_NULL, SCHEMA_ACTION_CREATE_STABLE, @@ -211,13 +227,10 @@ typedef struct { extern int64_t smlFactorNS[]; extern int64_t smlFactorS[]; -typedef int32_t (*_equal_fn_sml)(const void *, const void *); - int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle); void smlDestroyInfo(SSmlHandle *info); int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); -int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); -bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); +int smlJsonParseObj(char **start, char *end, SSmlLineInfo *element, int8_t *offset); void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); @@ -246,7 +259,8 @@ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements); int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements); int32_t smlJoinMeasureTag(SSmlLineInfo *elements); void smlBuildTsKv(SSmlKv *kv, int64_t ts); -int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); +int32_t smlParseEndTelnetJsonFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); +int32_t smlParseEndTelnetJsonUnFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs); static inline bool smlDoubleToInt64OverFlow(double num) { diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 80f583bbee..25abeb52fb 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -104,6 +104,9 @@ kvVal->type = TSDB_DATA_TYPE_UTINYINT; \ kvVal->u = result; +#define IS_COMMENT(protocol,data) \ + (protocol == TSDB_SML_LINE_PROTOCOL && data == '#') + int64_t smlToMilli[] = {3600000LL, 60000LL, 1000LL}; int64_t smlFactorNS[] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; int64_t smlFactorS[] = {1000LL, 1000000LL, 1000000000LL}; @@ -224,6 +227,7 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml if (code != TSDB_CODE_SUCCESS) { info->dataFormat = false; info->reRun = true; + taosMemoryFreeClear(pTableMeta); return code; } code = smlBuildSTableMeta(info->dataFormat, sMeta); @@ -305,7 +309,6 @@ bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { goto END; } SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); - if (maxKV == NULL) { goto END; } @@ -380,49 +383,23 @@ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen); SSmlTableInfo *tinfo = NULL; if (unlikely(oneTable == NULL)) { - code = smlBuildTableInfo(1, elements->measure, elements->measureLen, &tinfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - code = taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES); - if (code != 0) { - smlDestroyTableInfo(&tinfo); - return code; - } + SML_CHECK_CODE(smlBuildTableInfo(1, elements->measure, elements->measureLen, &tinfo)); + SML_CHECK_CODE(taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES)); tinfo->tags = taosArrayDup(info->preLineTagKV, NULL); - if (tinfo->tags == NULL) { - smlDestroyTableInfo(&tinfo); - return terrno; - } + SML_CHECK_NULL(tinfo->tags); for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i); - if (kv == NULL) { - smlDestroyTableInfo(&tinfo); - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); if (kv->keyEscaped) kv->key = NULL; if (kv->valueEscaped) kv->value = NULL; } - code = smlSetCTableName(tinfo, info->tbnameKey); - if (code != TSDB_CODE_SUCCESS) { - smlDestroyTableInfo(&tinfo); - return code; - } - code = getTableUid(info, elements, tinfo); - if (code != TSDB_CODE_SUCCESS) { - smlDestroyTableInfo(&tinfo); - return code; - } + SML_CHECK_CODE(smlSetCTableName(tinfo, info->tbnameKey)); + SML_CHECK_CODE(getTableUid(info, elements, tinfo)); if (info->dataFormat) { info->currSTableMeta->uid = tinfo->uid; - code = smlInitTableDataCtx(info->pQuery, info->currSTableMeta, &tinfo->tableDataCtx); - if (code != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - smlDestroyTableInfo(&tinfo); - return code; - } + SML_CHECK_CODE(smlInitTableDataCtx(info->pQuery, info->currSTableMeta, &tinfo->tableDataCtx)); } } else { tinfo = *oneTable; @@ -433,41 +410,41 @@ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { } if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; return TSDB_CODE_SUCCESS; + +END: + smlDestroyTableInfo(&tinfo); + return code; } -int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv) { - if (info->dataFormat) { - uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format true, ts:%" PRId64, info->id, kvTs->i); - int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1); - } - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildRow(info->currTableDataCtx); - } - clearColValArraySml(info->currTableDataCtx->pValues); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); - return ret; - } - } else { - uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i); - if (elements->colArray == NULL) { - elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); - if (elements->colArray == NULL) { - return terrno; - } - } - if (taosArrayPush(elements->colArray, kvTs) == NULL) { - return terrno; - } - if (taosArrayPush(elements->colArray, kv) == NULL) { - return terrno; - } +int32_t smlParseEndTelnetJsonFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv) { + int32_t code = 0; + uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format true, ts:%" PRId64, info->id, kvTs->i); + SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0)); + SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1)); + SML_CHECK_CODE(smlBuildRow(info->currTableDataCtx)); + + info->preLine = *elements; +END: + clearColValArraySml(info->currTableDataCtx->pValues); + if (unlikely(code != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); } + return code; +} + +int32_t smlParseEndTelnetJsonUnFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv) { + int32_t code = 0; + uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i); + if (elements->colArray == NULL) { + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + SML_CHECK_NULL(elements->colArray); + } + SML_CHECK_NULL(taosArrayPush(elements->colArray, kvTs)); + SML_CHECK_NULL (taosArrayPush(elements->colArray, kv)); info->preLine = *elements; - return TSDB_CODE_SUCCESS; +END: + return code; } int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) { @@ -494,15 +471,14 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) } static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnameKey) { + int32_t code = 0; bool autoChildName = false; size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter); if (delimiter > 0 && tbnameKey == NULL) { size_t totalNameLen = delimiter * (taosArrayGetSize(tags) - 1); for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - if (tag == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(tag); totalNameLen += tag->length; } if (totalNameLen < TSDB_TABLE_NAME_LEN) { @@ -513,9 +489,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - if (tag == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(tag); (void)strncat(childTableName, tag->value, TMIN(tag->length, TSDB_TABLE_NAME_LEN - 1 - strlen(childTableName))); if (i != taosArrayGetSize(tags) - 1) { (void)strncat(childTableName, tsSmlAutoChildTableNameDelimiter, TSDB_TABLE_NAME_LEN - 1 - strlen(childTableName)); @@ -533,9 +507,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); - if (tag == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(tag); // handle child table name if (childTableNameLen == tag->keyLen && strncmp(tag->key, tbnameKey, tag->keyLen) == 0) { (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); @@ -549,24 +521,22 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam } } - return TSDB_CODE_SUCCESS; +END: + return code; } int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { - int32_t code = smlParseTableName(oneTable->tags, oneTable->childTableName, tbnameKey); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + int32_t code = 0; + SArray *dst = NULL; + SML_CHECK_CODE(smlParseTableName(oneTable->tags, oneTable->childTableName, tbnameKey)); if (strlen(oneTable->childTableName) == 0) { - SArray *dst = taosArrayDup(oneTable->tags, NULL); - if (dst == NULL) { - return terrno; - } + dst = taosArrayDup(oneTable->tags, NULL); + SML_CHECK_NULL(dst); if (oneTable->sTableNameLen >= TSDB_TABLE_NAME_LEN) { uError("SML:smlSetCTableName super table name is too long"); - taosArrayDestroy(dst); - return TSDB_CODE_SML_INTERNAL_ERROR; + code = TSDB_CODE_SML_INTERNAL_ERROR; + goto END; } char superName[TSDB_TABLE_NAME_LEN] = {0}; RandTableName rName = {dst, NULL, (uint8_t)oneTable->sTableNameLen, oneTable->childTableName}; @@ -578,13 +548,12 @@ int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { rName.stbFullName = oneTable->sTableName; } - code = buildChildTableName(&rName); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - taosArrayDestroy(dst); + SML_CHECK_CODE(buildChildTableName(&rName)); } - return TSDB_CODE_SUCCESS; + +END: + taosArrayDestroy(dst); + return code; } int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo) { @@ -608,40 +577,25 @@ int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo * } int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta **sMeta) { + int32_t code = 0; SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); - if (!meta) { - return terrno; - } - + SML_CHECK_NULL(meta); if (unlikely(!isDataFormat)) { meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->tagHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } - + SML_CHECK_NULL(meta->tagHash); meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->colHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } + SML_CHECK_NULL(meta->colHash); } meta->tags = taosArrayInit(32, sizeof(SSmlKv)); - if (meta->tags == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } + SML_CHECK_NULL(meta->tags); meta->cols = taosArrayInit(32, sizeof(SSmlKv)); - if (meta->cols == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } + SML_CHECK_NULL(meta->cols); *sMeta = meta; return TSDB_CODE_SUCCESS; -cleanup: +END: smlDestroySTableMeta(meta); return TSDB_CODE_OUT_OF_MEMORY; } @@ -720,104 +674,6 @@ int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { return true; } -bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { - const char *pVal = kvVal->value; - int32_t len = kvVal->length; - char *endptr = NULL; - double result = taosStr2Double(pVal, &endptr); - if (pVal == endptr) { - smlBuildInvalidDataMsg(msg, "invalid data", pVal); - return false; - } - - int32_t left = len - (endptr - pVal); - if (left == 0 || (left == 3 && strncasecmp(endptr, "f64", left) == 0)) { - kvVal->type = TSDB_DATA_TYPE_DOUBLE; - kvVal->d = result; - } else if ((left == 3 && strncasecmp(endptr, "f32", left) == 0)) { - if (!IS_VALID_FLOAT(result)) { - smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_FLOAT; - kvVal->f = (float)result; - } else if ((left == 1 && *endptr == 'i') || (left == 3 && strncasecmp(endptr, "i64", left) == 0)) { - if (smlDoubleToInt64OverFlow(result)) { - errno = 0; - int64_t tmp = taosStr2Int64(pVal, &endptr, 10); - if (errno == ERANGE) { - smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_BIGINT; - kvVal->i = tmp; - return true; - } - kvVal->type = TSDB_DATA_TYPE_BIGINT; - kvVal->i = (int64_t)result; - } else if ((left == 1 && *endptr == 'u') || (left == 3 && strncasecmp(endptr, "u64", left) == 0)) { - if (result >= (double)UINT64_MAX || result < 0) { - errno = 0; - uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); - if (errno == ERANGE || result < 0) { - smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_UBIGINT; - kvVal->u = tmp; - return true; - } - kvVal->type = TSDB_DATA_TYPE_UBIGINT; - kvVal->u = result; - } else if (left == 3 && strncasecmp(endptr, "i32", left) == 0) { - if (!IS_VALID_INT(result)) { - smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_INT; - kvVal->i = result; - } else if (left == 3 && strncasecmp(endptr, "u32", left) == 0) { - if (!IS_VALID_UINT(result)) { - smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_UINT; - kvVal->u = result; - } else if (left == 3 && strncasecmp(endptr, "i16", left) == 0) { - if (!IS_VALID_SMALLINT(result)) { - smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_SMALLINT; - kvVal->i = result; - } else if (left == 3 && strncasecmp(endptr, "u16", left) == 0) { - if (!IS_VALID_USMALLINT(result)) { - smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_USMALLINT; - kvVal->u = result; - } else if (left == 2 && strncasecmp(endptr, "i8", left) == 0) { - if (!IS_VALID_TINYINT(result)) { - smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_TINYINT; - kvVal->i = result; - } else if (left == 2 && strncasecmp(endptr, "u8", left) == 0) { - if (!IS_VALID_UTINYINT(result)) { - smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_UTINYINT; - kvVal->u = result; - } else { - smlBuildInvalidDataMsg(msg, "invalid data", pVal); - return false; - } - return true; -} - int32_t smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen, STableMeta **pTableMeta) { *pTableMeta = NULL; @@ -832,11 +688,7 @@ int32_t smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen, ST (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); (void)memcpy(pName.tname, measure, measureLen); - int32_t code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - return TSDB_CODE_SUCCESS; + return catalogGetSTableMeta(info->pCatalog, &conn, &pName, pTableMeta); } static int64_t smlGenId() { @@ -913,53 +765,34 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH for (int j = 0; j < taosArrayGetSize(cols); ++j) { if (j == 0 && !isTag) continue; SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); - if (kv == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } - code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + SML_CHECK_NULL(kv); + SML_CHECK_CODE(smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info)); } for (int j = 0; j < taosArrayGetSize(checkDumplicateCols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(checkDumplicateCols, j); - if (kv == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); if (taosHashGet(schemaHash, kv->key, kv->keyLen) != NULL) { - return TSDB_CODE_PAR_DUPLICATED_COLUMN; + code = TSDB_CODE_PAR_DUPLICATED_COLUMN; + goto END; } } - return TSDB_CODE_SUCCESS; +END: + return code; } static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (hashTmp == NULL) { - code = terrno; - goto END; - } + SML_CHECK_NULL(hashTmp); int32_t i = 0; for (; i < length; i++) { - code = taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); - if (code != 0) { - goto END; - } - } - - if (isTag) { - i = 0; - } else { - i = 1; + SML_CHECK_CODE(taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES)); } + i = isTag ? 0 : 1; for (; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - if (kv == NULL) { - code = terrno; - goto END; - } + SML_CHECK_NULL(kv); if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { code = TSDB_CODE_SML_INVALID_DATA; goto END; @@ -982,36 +815,29 @@ static int32_t getBytes(uint8_t type, int32_t length) { static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray *results, int32_t numOfCols, bool isTag) { + int32_t code = TSDB_CODE_SUCCESS; for (int j = 0; j < taosArrayGetSize(cols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); - if (kv == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); ESchemaAction action = SCHEMA_ACTION_NULL; - int code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); - if (code != 0) { - return code; - } + SML_CHECK_CODE(smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info)); if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) { SField field = {0}; field.type = kv->type; field.bytes = getBytes(kv->type, kv->length); - (void)memcpy(field.name, kv->key, kv->keyLen); - if (taosArrayPush(results, &field) == NULL) { - return terrno; - } + (void)memcpy(field.name, kv->key, MIN(kv->keyLen, sizeof(field.name) - 1)); + SML_CHECK_NULL(taosArrayPush(results, &field)); } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); if (index == NULL) { uError("smlBuildFieldsList get error, key:%s", kv->key); - return TSDB_CODE_SML_INVALID_DATA; + code = TSDB_CODE_SML_INVALID_DATA; + goto END; } uint16_t newIndex = *index; if (isTag) newIndex -= numOfCols; SField *field = (SField *)taosArrayGet(results, newIndex); - if (field == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(field); field->bytes = getBytes(kv->type, kv->length); } } @@ -1020,18 +846,23 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO int32_t len = 0; for (int j = 0; j < taosArrayGetSize(results); ++j) { SField *field = taosArrayGet(results, j); - if (field == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(field); len += field->bytes; } if (len > maxLen) { return isTag ? TSDB_CODE_PAR_INVALID_TAGS_LENGTH : TSDB_CODE_PAR_INVALID_ROW_LENGTH; } - return TSDB_CODE_SUCCESS; +END: + return code; } +static FORCE_INLINE void smlBuildCreateStbReq(SMCreateStbReq *pReq, int32_t colVer, int32_t tagVer, tb_uid_t suid, int8_t source){ + pReq->colVer = colVer; + pReq->tagVer = tagVer; + pReq->suid = suid; + pReq->source = source; +} static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta, ESchemaAction action) { SRequestObj *pRequest = NULL; @@ -1046,58 +877,37 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pReq.numOfTags = taosArrayGetSize(pTags); pReq.pColumns = taosArrayInit(pReq.numOfColumns, sizeof(SFieldWithOptions)); - if (pReq.pColumns == NULL) { - code = terrno; - goto end; - } + SML_CHECK_NULL(pReq.pColumns); for (int32_t i = 0; i < pReq.numOfColumns; ++i) { SField *pField = taosArrayGet(pColumns, i); - if (pField == NULL) { - code = terrno; - goto end; - } + SML_CHECK_NULL(pField); SFieldWithOptions fieldWithOption = {0}; setFieldWithOptions(&fieldWithOption, pField); setDefaultOptionsForField(&fieldWithOption); - if (taosArrayPush(pReq.pColumns, &fieldWithOption) == NULL) { - code = terrno; - goto end; - } + SML_CHECK_NULL(taosArrayPush(pReq.pColumns, &fieldWithOption)); } if (action == SCHEMA_ACTION_CREATE_STABLE) { - pReq.colVer = 1; - pReq.tagVer = 1; - pReq.suid = 0; - pReq.source = TD_REQ_FROM_APP; pSql = "sml_create_stable"; + smlBuildCreateStbReq(&pReq, 1, 1, 0, TD_REQ_FROM_APP); } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { - pReq.colVer = pTableMeta->sversion; - pReq.tagVer = pTableMeta->tversion + 1; - pReq.suid = pTableMeta->uid; - pReq.source = TD_REQ_FROM_TAOX; pSql = (action == SCHEMA_ACTION_ADD_TAG) ? "sml_add_tag" : "sml_modify_tag_size"; + smlBuildCreateStbReq(&pReq, pTableMeta->sversion, pTableMeta->tversion + 1, pTableMeta->uid, TD_REQ_FROM_TAOX); } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) { - pReq.colVer = pTableMeta->sversion + 1; - pReq.tagVer = pTableMeta->tversion; - pReq.suid = pTableMeta->uid; - pReq.source = TD_REQ_FROM_TAOX; pSql = (action == SCHEMA_ACTION_ADD_COLUMN) ? "sml_add_column" : "sml_modify_column_size"; + smlBuildCreateStbReq(&pReq, pTableMeta->sversion + 1, pTableMeta->tversion, pTableMeta->uid, TD_REQ_FROM_TAOX); } else { uError("SML:0x%" PRIx64 " invalid action:%d", info->id, action); code = TSDB_CODE_SML_INVALID_DATA; - goto end; + goto END; } - code = buildRequest(info->taos->id, pSql, strlen(pSql), NULL, false, &pRequest, 0); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } + SML_CHECK_CODE(buildRequest(info->taos->id, pSql, strlen(pSql), NULL, false, &pRequest, 0)); pRequest->syncQuery = true; if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto end; + goto END; } if (pReq.numOfTags == 0) { @@ -1106,40 +916,29 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, field.type = TSDB_DATA_TYPE_NCHAR; field.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; tstrncpy(field.name, tsSmlTagName, sizeof(field.name)); - if (taosArrayPush(pReq.pTags, &field) == NULL) { - code = terrno; - goto end; - } + SML_CHECK_NULL(taosArrayPush(pReq.pTags, &field)); } pReq.commentLen = -1; pReq.igExists = true; - code = tNameExtractFullName(pName, pReq.name); - if (TSDB_CODE_SUCCESS != code) { - goto end; - } + SML_CHECK_CODE(tNameExtractFullName(pName, pReq.name)); pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); pCmdMsg.msgType = TDMT_MND_CREATE_STB; pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); if (pCmdMsg.msgLen < 0) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; + code = pCmdMsg.msgLen; + goto END; } pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); - if (NULL == pCmdMsg.pMsg) { - code = terrno; - goto end; - } - - if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) < 0) { - code = TSDB_CODE_OUT_OF_MEMORY; + SML_CHECK_NULL(pCmdMsg.pMsg); + code = tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); + if (code < 0) { taosMemoryFree(pCmdMsg.pMsg); - goto end; + goto END; } SQuery pQuery = {0}; - (void)memset(&pQuery, 0, sizeof(pQuery)); pQuery.execMode = QUERY_EXEC_MODE_RPC; pQuery.pCmdMsg = &pCmdMsg; pQuery.msgType = pQuery.pCmdMsg->msgType; @@ -1148,19 +947,134 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, launchQueryImpl(pRequest, &pQuery, true, NULL); // no need to check return value if (pRequest->code == TSDB_CODE_SUCCESS) { - code = catalogRemoveTableMeta(info->pCatalog, pName); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } + SML_CHECK_CODE(catalogRemoveTableMeta(info->pCatalog, pName)); } code = pRequest->code; -end: +END: destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); return code; } +static int32_t smlCreateTable(SSmlHandle *info, SRequestConnInfo *conn, SSmlSTableMeta *sTableData, + SName *pName, STableMeta **pTableMeta){ + int32_t code = 0; + SArray *pColumns = NULL; + SArray *pTags = NULL; + SML_CHECK_CODE(smlCheckAuth(info, conn, NULL, AUTH_TYPE_WRITE)); + uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName->tname); + pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); + SML_CHECK_NULL(pColumns); + pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); + SML_CHECK_NULL(pTags); + SML_CHECK_CODE(smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true)); + SML_CHECK_CODE(smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false)); + pTags = NULL; + SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE)); + info->cost.numOfCreateSTables++; + taosMemoryFreeClear(*pTableMeta); + + SML_CHECK_CODE(catalogGetSTableMeta(info->pCatalog, conn, pName, pTableMeta)); + +END: + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); + return code; +} + +static int32_t smlBuildFields(SArray **pColumns, SArray **pTags, STableMeta *pTableMeta, SSmlSTableMeta *sTableData){ + int32_t code = 0; + *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + (pTableMeta)->tableInfo.numOfColumns, sizeof(SField)); + SML_CHECK_NULL(pColumns); + *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + (pTableMeta)->tableInfo.numOfTags, sizeof(SField)); + SML_CHECK_NULL(pTags); + for (uint16_t i = 0; i < (pTableMeta)->tableInfo.numOfColumns + (pTableMeta)->tableInfo.numOfTags; i++) { + SField field = {0}; + field.type = (pTableMeta)->schema[i].type; + field.bytes = (pTableMeta)->schema[i].bytes; + tstrncpy(field.name, (pTableMeta)->schema[i].name, sizeof(field.name)); + if (i < (pTableMeta)->tableInfo.numOfColumns) { + SML_CHECK_NULL(taosArrayPush(*pColumns, &field)); + } else { + SML_CHECK_NULL(taosArrayPush(*pTags, &field)); + } + } +END: + return code; +} +static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn, + SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){ + ESchemaAction action = SCHEMA_ACTION_NULL; + SArray *pColumns = NULL; + SArray *pTags = NULL; + int32_t code = 0; + SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, sTableData->cols, &action, true)); + + if (action != SCHEMA_ACTION_NULL) { + SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE)); + uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName->tname, + action); + SML_CHECK_CODE(smlBuildFields(&pColumns, &pTags, *pTableMeta, sTableData)); + SML_CHECK_CODE(smlBuildFieldsList(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, pTags, + (*pTableMeta)->tableInfo.numOfColumns, true)); + + pTags = NULL; + SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, pTags, (*pTableMeta), action)); + + info->cost.numOfAlterTagSTables++; + taosMemoryFreeClear(*pTableMeta); + SML_CHECK_CODE(catalogRefreshTableMeta(info->pCatalog, conn, pName, -1)); + SML_CHECK_CODE(catalogGetSTableMeta(info->pCatalog, conn, pName, pTableMeta)); + } + +END: + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); + return code; +} + +static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn, + SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){ + ESchemaAction action = SCHEMA_ACTION_NULL; + SArray *pColumns = NULL; + SArray *pTags = NULL; + int32_t code = 0; + SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false)); + + if (action != SCHEMA_ACTION_NULL) { + SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE)); + uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName->tname, + action); + SML_CHECK_CODE(smlBuildFields(&pColumns, &pTags, *pTableMeta, sTableData)); + SML_CHECK_CODE(smlBuildFieldsList(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, pColumns, + (*pTableMeta)->tableInfo.numOfColumns, false)); + + pTags = NULL; + SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, pTags, (*pTableMeta), action)); + + info->cost.numOfAlterColSTables++; + taosMemoryFreeClear(*pTableMeta); + SML_CHECK_CODE(catalogRefreshTableMeta(info->pCatalog, conn, pName, -1)); + SML_CHECK_CODE(catalogGetSTableMeta(info->pCatalog, conn, pName, pTableMeta)); + } + + END: + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); + return code; +} + +static int32_t smlBuildTempHash(SHashObj *hashTmp, STableMeta *pTableMeta, uint16_t start, uint16_t end){ + int32_t code = 0; + for (uint16_t i = start; i < end; i++) { + SML_CHECK_CODE(taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES)); + } + +END: + return code; +} + static int32_t smlModifyDBSchemas(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas start, format:%d, needModifySchema:%d", info->id, info->dataFormat, info->needModifySchema); @@ -1188,286 +1102,43 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { size_t superTableLen = 0; void *superTable = taosHashGetKey(tmp, &superTableLen); char *measure = taosMemoryMalloc(superTableLen); - if (measure == NULL) { - code = terrno; - goto end; - } + SML_CHECK_NULL(measure); (void)memcpy(measure, superTable, superTableLen); PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); smlStrReplace(measure, superTableLen); (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - (void)memcpy(pName.tname, measure, superTableLen); + (void)memcpy(pName.tname, measure, MIN(superTableLen, TSDB_TABLE_NAME_LEN - 1)); taosMemoryFree(measure); code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { - code = smlCheckAuth(info, &conn, NULL, AUTH_TYPE_WRITE); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname); - SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); - if (pColumns == NULL) { - code = terrno; - goto end; - } - SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); - if (pTags == NULL) { - code = terrno; - taosArrayDestroy(pColumns); - goto end; - } - code = smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBuildFieldsList tag1 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - code = smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBuildFieldsList col1 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); - taosArrayDestroy(pColumns); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - info->cost.numOfCreateSTables++; - taosMemoryFreeClear(pTableMeta); - - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); - goto end; - } + SML_CHECK_CODE(smlCreateTable(info, &conn, sTableData, &pName, &pTableMeta)); } else if (code == TSDB_CODE_SUCCESS) { if (smlIsPKTable(pTableMeta)) { code = TSDB_CODE_SML_NOT_SUPPORT_PK; - goto end; - } - - hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, - HASH_NO_LOCK); - if (hashTmp == NULL) { - code = terrno; - goto end; - } - for (uint16_t i = pTableMeta->tableInfo.numOfColumns; - i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - code = taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); - if (code != 0) { - goto end; - } - } - - ESchemaAction action = SCHEMA_ACTION_NULL; - code = - smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, sTableData->cols, &action, true); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - if (action != SCHEMA_ACTION_NULL) { - code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName.tname, - action); - SArray *pColumns = - taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - if (pColumns == NULL) { - code = terrno; - goto end; - } - SArray *pTags = - taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - if (pTags == NULL) { - taosArrayDestroy(pColumns); - code = terrno; - goto end; - } - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - SField field = {0}; - field.type = pTableMeta->schema[i].type; - field.bytes = pTableMeta->schema[i].bytes; - tstrncpy(field.name, pTableMeta->schema[i].name, sizeof(field.name)); - if (i < pTableMeta->tableInfo.numOfColumns) { - if (taosArrayPush(pColumns, &field) == NULL) { - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - code = terrno; - goto end; - } - } else { - if (taosArrayPush(pTags, &field) == NULL) { - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - code = terrno; - goto end; - } - } - } - code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, - pTableMeta->tableInfo.numOfColumns, true); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBuildFieldsList tag2 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - - if (taosArrayGetSize(pTags) + pTableMeta->tableInfo.numOfColumns > TSDB_MAX_COLUMNS) { - uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); - code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - if (taosArrayGetSize(pTags) > TSDB_MAX_TAGS) { - uError("SML:0x%" PRIx64 " too many tags than 128", info->id); - code = TSDB_CODE_PAR_INVALID_TAGS_NUM; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - taosArrayDestroy(pColumns); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - - info->cost.numOfAlterTagSTables++; - taosMemoryFreeClear(pTableMeta); - code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } + goto END; } + hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SML_CHECK_NULL(hashTmp); + SML_CHECK_CODE(smlBuildTempHash(hashTmp, pTableMeta, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags)); + SML_CHECK_CODE(smlModifyTag(info, hashTmp, &conn, sTableData, &pName, &pTableMeta)); taosHashClear(hashTmp); - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - code = taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - } - action = SCHEMA_ACTION_NULL; - code = - smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - if (action != SCHEMA_ACTION_NULL) { - code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName.tname, - action); - SArray *pColumns = - taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - if (pColumns == NULL) { - code = terrno; - goto end; - } - SArray *pTags = - taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - if (pTags == NULL) { - taosArrayDestroy(pColumns); - code = terrno; - goto end; - } - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - SField field = {0}; - field.type = pTableMeta->schema[i].type; - field.bytes = pTableMeta->schema[i].bytes; - tstrncpy(field.name, pTableMeta->schema[i].name, sizeof(field.name)); - if (i < pTableMeta->tableInfo.numOfColumns) { - if (taosArrayPush(pColumns, &field) == NULL) { - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - code = terrno; - goto end; - } - } else { - if (taosArrayPush(pTags, &field) == NULL) { - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - code = terrno; - goto end; - } - } - } - - code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, - pTableMeta->tableInfo.numOfColumns, false); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBuildFieldsList col2 failed. %s", info->id, pName.tname); - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - - if (taosArrayGetSize(pColumns) + pTableMeta->tableInfo.numOfTags > TSDB_MAX_COLUMNS) { - uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); - code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; - taosArrayDestroy(pColumns); - taosArrayDestroy(pTags); - goto end; - } - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - taosArrayDestroy(pColumns); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - - info->cost.numOfAlterColSTables++; - taosMemoryFreeClear(pTableMeta); - code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); - goto end; - } - } + SML_CHECK_CODE(smlBuildTempHash(hashTmp, pTableMeta, 0, pTableMeta->tableInfo.numOfColumns)); + SML_CHECK_CODE(smlModifyCols(info, hashTmp, &conn, sTableData, &pName, &pTableMeta)); needCheckMeta = true; taosHashCleanup(hashTmp); hashTmp = NULL; } else { uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code)); - goto end; + goto END; } if (needCheckMeta) { - code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, - sTableData->tags, true); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, pName.tname); - goto end; - } - code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, pName.tname); - goto end; - } + SML_CHECK_CODE(smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, sTableData->tags, true)); + SML_CHECK_CODE(smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false)); } taosMemoryFreeClear(sTableData->tableMeta); @@ -1481,7 +1152,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { return TSDB_CODE_SUCCESS; -end: +END: taosHashCancelIterate(info->superTables, tmp); taosHashCleanup(hashTmp); taosMemoryFreeClear(pTableMeta); @@ -1493,40 +1164,37 @@ end: } static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SHashObj *checkDuplicate) { + int32_t code = 0; terrno = 0; for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - if (kv == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); if (ret == 0) { - if (taosArrayPush(metaArray, kv) == NULL) { - return terrno; - } + SML_CHECK_NULL(taosArrayPush(metaArray, kv)); if (taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { - return TSDB_CODE_PAR_DUPLICATED_COLUMN; + code = TSDB_CODE_PAR_DUPLICATED_COLUMN; + goto END; } } else if (terrno == TSDB_CODE_DUP_KEY) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; } } - return TSDB_CODE_SUCCESS; + +END: + return code; } static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg, SHashObj *checkDuplicate) { + int32_t code = 0; for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - if (kv == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); if (index) { SSmlKv *value = (SSmlKv *)taosArrayGet(metaArray, *index); - if (value == NULL) { - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(value); if (isTag) { if (kv->length > value->length) { @@ -1536,7 +1204,8 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols } if (kv->type != value->type) { smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); - return TSDB_CODE_SML_NOT_SAME_TYPE; + code = TSDB_CODE_SML_NOT_SAME_TYPE; + goto END; } if (IS_VAR_DATA_TYPE(kv->type) && (kv->length > value->length)) { // update string len, if bigger @@ -1547,24 +1216,21 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols if (tmp > INT16_MAX) { smlBuildInvalidDataMsg(msg, "too many cols or tags", kv->key); uError("too many cols or tags"); - return TSDB_CODE_SML_INVALID_DATA; + code = TSDB_CODE_SML_INVALID_DATA; + goto END; } int16_t size = tmp; - int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); - if (ret == 0) { - if (taosArrayPush(metaArray, kv) == NULL) { - return terrno; - } - if (taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { - return TSDB_CODE_PAR_DUPLICATED_COLUMN; - } - } else { - return ret; + SML_CHECK_CODE(taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES)); + SML_CHECK_NULL(taosArrayPush(metaArray, kv)); + if (taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { + code = TSDB_CODE_PAR_DUPLICATED_COLUMN; + goto END; } } } - return TSDB_CODE_SUCCESS; +END: + return code; } void smlDestroyTableInfo(void *para) { @@ -1631,94 +1297,71 @@ void smlDestroyInfo(SSmlHandle *info) { int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { int32_t code = TSDB_CODE_SUCCESS; SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); - if (NULL == info) { - return terrno; - } - if (taos != NULL) { + SML_CHECK_NULL(info); + if (taos != NULL){ info->taos = acquireTscObj(*(int64_t *)taos); - if (info->taos == NULL) { - code = TSDB_CODE_TSC_DISCONNECTED; - goto FAILED; - } - code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); - goto FAILED; - } + SML_CHECK_NULL(info->taos); + SML_CHECK_CODE(catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog)); } info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + SML_CHECK_NULL(info->pVgHash); info->childTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SML_CHECK_NULL(info->childTables); info->tableUids = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SML_CHECK_NULL(info->tableUids); info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (info->pVgHash == NULL || info->childTables == NULL || info->superTables == NULL || info->tableUids == NULL) { - uError("create SSmlHandle hash obj failed"); - code = terrno; - goto FAILED; - } + SML_CHECK_NULL(info->superTables); taosHashSetFreeFp(info->superTables, smlDestroySTableMeta); taosHashSetFreeFp(info->childTables, smlDestroyTableInfo); info->id = smlGenId(); - code = smlInitHandle(&info->pQuery); - if (code != TSDB_CODE_SUCCESS) { - goto FAILED; - } + SML_CHECK_CODE(smlInitHandle(&info->pQuery)); info->dataFormat = true; - info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); + SML_CHECK_NULL(info->tagJsonArray); info->valueJsonArray = taosArrayInit(8, POINTER_BYTES); + SML_CHECK_NULL(info->valueJsonArray); info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); + SML_CHECK_NULL(info->preLineTagKV); info->escapedStringList = taosArrayInit(8, POINTER_BYTES); - if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || - info->preLineTagKV == NULL || info->escapedStringList == NULL) { - uError("SML:0x%" PRIx64 " failed to allocate memory", info->id); - code = terrno; - goto FAILED; - } + SML_CHECK_NULL(info->escapedStringList); *handle = info; return code; -FAILED: +END: smlDestroyInfo(info); return code; } static int32_t smlPushCols(SArray *colsArray, SArray *cols) { + int32_t code = TSDB_CODE_SUCCESS; SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (!kvHash) { - uError("SML:smlDealCols failed to allocate memory"); - return terrno; - } + SML_CHECK_NULL(kvHash); for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - if (kv == NULL) { - taosHashCleanup(kvHash); - return TSDB_CODE_SML_INVALID_DATA; - } + SML_CHECK_NULL(kv); terrno = 0; - int32_t code = taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + code = taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { - taosHashCleanup(kvHash); - return TSDB_CODE_PAR_DUPLICATED_COLUMN; - } - if (code != TSDB_CODE_SUCCESS) { - taosHashCleanup(kvHash); - return code; + code = TSDB_CODE_PAR_DUPLICATED_COLUMN; + goto END; } + SML_CHECK_CODE(code); } - if (taosArrayPush(colsArray, &kvHash) == NULL) { - taosHashCleanup(kvHash); - return terrno; - } - return TSDB_CODE_SUCCESS; + SML_CHECK_NULL(taosArrayPush(colsArray, &kvHash)); + return code; +END: + taosHashCleanup(kvHash); + return code; } static int32_t smlParseLineBottom(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlParseLineBottom start, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); + int32_t code = 0; if (info->dataFormat) return TSDB_CODE_SUCCESS; for (int32_t i = 0; i < info->lineNum; i++) { @@ -1728,10 +1371,6 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measure, elements->measureTagsLen); if (tmp) tinfo = *tmp; - } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, - elements->measureLen + elements->tagsLen); - if (tmp) tinfo = *tmp; } else { SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen); @@ -1754,81 +1393,62 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { return TSDB_CODE_PAR_TOO_MANY_COLUMNS; } - int ret = smlPushCols(tinfo->cols, elements->colArray); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } + SML_CHECK_CODE(smlPushCols(tinfo->cols, elements->colArray)); SSmlSTableMeta **tableMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); if (tableMeta) { // update meta uDebug("SML:0x%" PRIx64 " smlParseLineBottom update meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); - ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf, - (*tableMeta)->tagHash); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf, - (*tableMeta)->colHash); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlUpdateMeta failed, ret:%d", info->id, ret); - return ret; - } + SML_CHECK_CODE(smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf, + (*tableMeta)->tagHash)); + SML_CHECK_CODE(smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf, + (*tableMeta)->colHash)); } else { uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); SSmlSTableMeta *meta = NULL; - ret = smlBuildSTableMeta(info->dataFormat, &meta); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - ret = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); - if (ret != TSDB_CODE_SUCCESS) { + SML_CHECK_CODE(smlBuildSTableMeta(info->dataFormat, &meta)); + SML_CHECK_CODE(taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES)); + code = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS) { + smlDestroySTableMeta(meta); uError("SML:0x%" PRIx64 " put measuer to hash failed", info->id); - return ret; - } - ret = smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags, NULL); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlInsertMeta(meta->colHash, meta->cols, elements->colArray, meta->tagHash); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " insert meta failed:%s", info->id, tstrerror(ret)); - return ret; + goto END; } + SML_CHECK_CODE(smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags, NULL)); + SML_CHECK_CODE(smlInsertMeta(meta->colHash, meta->cols, elements->colArray, meta->tagHash)); } } uDebug("SML:0x%" PRIx64 " smlParseLineBottom end, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); - return TSDB_CODE_SUCCESS; +END: + return code; } static int32_t smlInsertData(SSmlHandle *info) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + char *measure = NULL; + SSmlTableInfo **oneTable = NULL; uDebug("SML:0x%" PRIx64 " smlInsertData start, format:%d", info->id, info->dataFormat); if (info->pRequest->dbList == NULL) { info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN); - if (info->pRequest->dbList == NULL) { - return terrno; - } + SML_CHECK_NULL(info->pRequest->dbList); } char *data = (char *)taosArrayReserve(info->pRequest->dbList, 1); - if (data == NULL) { - return terrno; - } + SML_CHECK_NULL(data); SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); (void)tNameGetFullDbName(&pName, data); // ignore - SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); + oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); while (oneTable) { SSmlTableInfo *tableData = *oneTable; int measureLen = tableData->sTableNameLen; - char *measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); - if (measure == NULL) { - return terrno; - } + measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); + SML_CHECK_NULL(measure); (void)memcpy(measure, tableData->sTableName, tableData->sTableNameLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); smlStrReplace(measure, measureLen); @@ -1837,14 +1457,9 @@ static int32_t smlInsertData(SSmlHandle *info) { if (info->pRequest->tableList == NULL) { info->pRequest->tableList = taosArrayInit(1, sizeof(SName)); - if (info->pRequest->tableList == NULL) { - return terrno; - } + SML_CHECK_NULL(info->pRequest->tableList); } - if (taosArrayPush(info->pRequest->tableList, &pName) == NULL) { - return terrno; - } - + SML_CHECK_NULL(taosArrayPush(info->pRequest->tableList, &pName)); tstrncpy(pName.tname, tableData->childTableName, sizeof(pName.tname)); SRequestConnInfo conn = {0}; @@ -1853,36 +1468,18 @@ static int32_t smlInsertData(SSmlHandle *info) { conn.requestObjRefId = info->pRequest->self; conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFree(measure); - taosHashCancelIterate(info->childTables, oneTable); - return code; - } + SML_CHECK_CODE(smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE)); - SVgroupInfo vg; - code = catalogGetTableHashVgroup(info->pCatalog, &conn, &pName, &vg); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetTableHashVgroup failed. table name: %s", info->id, tableData->childTableName); - taosMemoryFree(measure); - taosHashCancelIterate(info->childTables, oneTable); - return code; - } - code = taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " taosHashPut failed. table name: %s", info->id, tableData->childTableName); - taosMemoryFree(measure); - taosHashCancelIterate(info->childTables, oneTable); - return code; - } + SVgroupInfo vg = {0}; + SML_CHECK_CODE(catalogGetTableHashVgroup(info->pCatalog, &conn, &pName, &vg)); + SML_CHECK_CODE(taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg))); SSmlSTableMeta **pMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); - if (unlikely(NULL == pMeta || NULL == (*pMeta)->tableMeta)) { + if (unlikely(NULL == pMeta || NULL == *pMeta || NULL == (*pMeta)->tableMeta)) { uError("SML:0x%" PRIx64 " NULL == pMeta. table name: %s", info->id, tableData->childTableName); - taosMemoryFree(measure); - taosHashCancelIterate(info->childTables, oneTable); - return TSDB_CODE_SML_INTERNAL_ERROR; + code = TSDB_CODE_SML_INTERNAL_ERROR; + goto END; } // use tablemeta of stable to save vgid and uid of child table @@ -1891,23 +1488,14 @@ static int32_t smlInsertData(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname, tableData->uid, info->dataFormat); - code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, + SML_CHECK_CODE(smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, (*pMeta)->tableMeta, tableData->childTableName, measure, measureLen, info->ttl, info->msgBuf.buf, - info->msgBuf.len); - taosMemoryFree(measure); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBindData failed", info->id); - taosHashCancelIterate(info->childTables, oneTable); - return code; - } + info->msgBuf.len)); + taosMemoryFreeClear(measure); oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); } - code = smlBuildOutput(info->pQuery, info->pVgHash); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlBuildOutput failed", info->id); - return code; - } + SML_CHECK_CODE(smlBuildOutput(info->pQuery, info->pVgHash)); info->cost.insertRpcTime = taosGetTimestampUs(); SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary; @@ -1919,6 +1507,11 @@ static int32_t smlInsertData(SSmlHandle *info) { tstrerror(info->pRequest->code)); return info->pRequest->code; + +END: + taosMemoryFree(measure); + taosHashCancelIterate(info->childTables, oneTable); + return code; } static void smlPrintStatisticInfo(SSmlHandle *info) { @@ -1958,11 +1551,33 @@ int32_t smlClearForRerun(SSmlHandle *info) { info->currTableDataCtx = NULL; SVnodeModifyOpStmt *stmt = (SVnodeModifyOpStmt *)(info->pQuery->pRoot); + if (stmt == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } stmt->freeHashFunc(stmt->pTableBlockHashObj); stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (stmt->pTableBlockHashObj == NULL) { + uError("SML:0x%" PRIx64 " stmt->pTableBlockHashObj == NULL", info->id); + return terrno; + } return TSDB_CODE_SUCCESS; } +static void printRaw(int64_t id, int lineNum, int numLines, ELogLevel level, char* data, int32_t len){ + char *print = taosMemoryCalloc(len + 1, 1); + if (print == NULL) { + uError("SML:0x%" PRIx64 " smlParseLine failed. code : %d", id, terrno); + return; + } + (void)memcpy(print, data, len); + if (level == DEBUG_DEBUG){ + uDebug("SML:0x%" PRIx64 " smlParseLine is raw, line %d/%d : %s", id, lineNum, numLines, print); + }else if (level == DEBUG_ERROR){ + uError("SML:0x%" PRIx64 " smlParseLine failed. line %d/%d : %s", id, lineNum, numLines, print); + } + taosMemoryFree(print); +} + static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLineEnd, int numLines, int i, char **tmp, int *len) { if (lines) { @@ -1976,24 +1591,15 @@ static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLi } (*len)++; } - if (info->protocol == TSDB_SML_LINE_PROTOCOL && (*tmp)[0] == '#') { // this line is comment + if (IS_COMMENT(info->protocol,(*tmp)[0])) { // this line is comment return false; } } if (*rawLine != NULL && (uDebugFlag & DEBUG_DEBUG)) { - char *print = taosMemoryCalloc(*len + 1, 1); - if (print != NULL) { - (void)memcpy(print, *tmp, *len); - uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id, numLines, - info->protocol, *len, print); - taosMemoryFree(print); - } else { - uError("SML:0x%" PRIx64 " smlParseLine taosMemoryCalloc failed", info->id); - } + printRaw(info->id, i, numLines, DEBUG_DEBUG, *tmp, *len); } else { - uDebug("SML:0x%" PRIx64 " smlParseLine is not numLines:%d, protocol:%d, len:%d, data:%s", info->id, numLines, - info->protocol, *len, *tmp); + uDebug("SML:0x%" PRIx64 " smlParseLine is not raw, line %d/%d : %s", info->id, i, numLines, *tmp); } return true; } @@ -2042,14 +1648,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } if (code != TSDB_CODE_SUCCESS) { if (rawLine != NULL) { - char *print = taosMemoryCalloc(len + 1, 1); - if (print == NULL) { - uError("SML:0x%" PRIx64 " smlParseLine failed. out of memory", info->id); - return code; - } - (void)memcpy(print, tmp, len); - uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, print); - taosMemoryFree(print); + printRaw(info->id, i, numLines, DEBUG_ERROR, tmp, len); } else { uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); } @@ -2078,21 +1677,12 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL info->cost.parseTime = taosGetTimestampUs(); - code = smlParseLine(info, lines, rawLine, rawLineEnd, numLines); - if (code != 0) { - uError("SML:0x%" PRIx64 " smlParseLine error : %s", info->id, tstrerror(code)); - return code; - } - code = smlParseLineBottom(info); - if (code != 0) { - uError("SML:0x%" PRIx64 " smlParseLineBottom error : %s", info->id, tstrerror(code)); - return code; - } + SML_CHECK_CODE(smlParseLine(info, lines, rawLine, rawLineEnd, numLines)); + SML_CHECK_CODE(smlParseLineBottom(info)); info->cost.lineNum = info->lineNum; info->cost.numOfSTables = taosHashGetSize(info->superTables); info->cost.numOfCTables = taosHashGetSize(info->childTables); - info->cost.schemaTime = taosGetTimestampUs(); do { @@ -2105,18 +1695,11 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL uInfo("SML:0x%" PRIx64 " smlModifyDBSchemas retry code:%s, times:%d", info->id, tstrerror(code), retryNum); } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES); - if (code != 0) { - uError("SML:0x%" PRIx64 " smlModifyDBSchemas error : %s", info->id, tstrerror(code)); - return code; - } - + SML_CHECK_CODE(code); info->cost.insertBindTime = taosGetTimestampUs(); - code = smlInsertData(info); - if (code != 0) { - uError("SML:0x%" PRIx64 " smlInsertData error : %s", info->id, tstrerror(code)); - return code; - } + SML_CHECK_CODE(smlInsertData(info)); +END: return code; } @@ -2160,29 +1743,16 @@ void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawL TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, char *rawLineEnd, int numLines, int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey) { - int32_t code = TSDB_CODE_SUCCESS; - if (NULL == taos) { - uError("SML:taos_schemaless_insert error taos is null"); - return NULL; - } + int32_t code = TSDB_CODE_SUCCESS; SRequestObj *request = NULL; - SSmlHandle *info = NULL; - int cnt = 0; + SSmlHandle *info = NULL; + int cnt = 0; while (1) { - code = createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid, &request); - if (TSDB_CODE_SUCCESS != code) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - + SML_CHECK_CODE(createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid, &request)); SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - code = smlBuildSmlInfo(taos, &info); - if (code != TSDB_CODE_SUCCESS) { - request->code = code; - smlBuildInvalidDataMsg(&msg, "init SSmlHandle failed", NULL); - uError("SML:taos_schemaless_insert error SSmlHandle is null, err msg:%s", tstrerror(code)); - goto end; - } + request->code = smlBuildSmlInfo(taos, &info); + SML_CHECK_CODE(request->code); + info->pRequest = request; info->pRequest->pQuery = info->pQuery; info->ttl = ttl; @@ -2198,20 +1768,20 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, if (request->pDb == NULL) { request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; smlBuildInvalidDataMsg(&msg, "Database not specified", NULL); - goto end; + goto END; } if (protocol < TSDB_SML_LINE_PROTOCOL || protocol > TSDB_SML_JSON_PROTOCOL) { request->code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; smlBuildInvalidDataMsg(&msg, "protocol invalidate", NULL); - goto end; + goto END; } if (protocol == TSDB_SML_LINE_PROTOCOL && (precision < TSDB_SML_TIMESTAMP_NOT_CONFIGURED || precision > TSDB_SML_TIMESTAMP_NANO_SECONDS)) { request->code = TSDB_CODE_SML_INVALID_PRECISION_TYPE; smlBuildInvalidDataMsg(&msg, "precision invalidate for line protocol", NULL); - goto end; + goto END; } if (protocol == TSDB_SML_JSON_PROTOCOL) { @@ -2219,7 +1789,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, } else if (numLines <= 0) { request->code = TSDB_CODE_SML_INVALID_DATA; smlBuildInvalidDataMsg(&msg, "line num is invalid", NULL); - goto end; + goto END; } code = smlProcess(info, lines, rawLine, rawLineEnd, numLines); @@ -2249,7 +1819,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, break; } -end: + END: smlDestroyInfo(info); return (TAOS_RES *)request; } @@ -2275,6 +1845,16 @@ end: TAOS_RES *taos_schemaless_insert_ttl_with_reqid_tbname_key(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey) { + if (taos == NULL || lines == NULL || numLines <= 0) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + for (int i = 0; i < numLines; i++){ + if (lines[i] == NULL){ + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + } return taos_schemaless_insert_inner(taos, lines, NULL, NULL, numLines, protocol, precision, ttl, reqid, tbnameKey); } @@ -2298,26 +1878,32 @@ TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLi reqid); } -static void getRawLineLen(char *lines, int len, int32_t *totalRows, int protocol) { +static int32_t getRawLineLen(char *lines, int len, int protocol) { int numLines = 0; - *totalRows = 0; char *tmp = lines; for (int i = 0; i < len; i++) { if (lines[i] == '\n' || i == len - 1) { - numLines++; - if (tmp[0] != '#' || protocol != TSDB_SML_LINE_PROTOCOL) { // ignore comment - (*totalRows)++; + if (!IS_COMMENT(protocol, tmp[0])) { // ignore comment + numLines++; } tmp = lines + i + 1; } } + return numLines; } TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid_tbname_key(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey) { - getRawLineLen(lines, len, totalRows, protocol); - return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid, + if (taos == NULL || lines == NULL || len <= 0) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + int numLines = getRawLineLen(lines, len, protocol); + if (totalRows != NULL){ + *totalRows = numLines; + } + return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, numLines, protocol, precision, ttl, reqid, tbnameKey); } diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index ece1ddf61f..9568074018 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -184,7 +184,7 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { return 0; } -int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { +int smlJsonParseObj(char **start, char *end, SSmlLineInfo *element, int8_t *offset) { int index = 0; while (*(*start)) { if ((*start)[0] != '"') { @@ -197,6 +197,10 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { return TSDB_CODE_TSC_INVALID_JSON; } + if ((*start) + offset[index] >= end){ + return TSDB_CODE_TSC_INVALID_JSON; + } + if ((*start)[1] == 'm') { (*start) += offset[index++]; element->measure = *start; @@ -539,28 +543,19 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ } static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { - int32_t ret = 0; + int32_t code = 0; if(info->dataFormat){ - ret = smlProcessSuperTable(info, elements); - if(ret != 0){ - if(info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; - } - } - ret = smlProcessTagJson(info, tags); - if(ret != 0){ - if(info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; - } - ret = smlJoinMeasureTag(elements); - if(ret != 0){ - return ret; + SML_CHECK_CODE(smlProcessSuperTable(info, elements)); } + SML_CHECK_CODE(smlProcessTagJson(info, tags)); + SML_CHECK_CODE(smlJoinMeasureTag(elements)); return smlProcessChildTable(info, elements); + +END: + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return code; } static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) { @@ -757,19 +752,18 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo } SSmlKv kvTs = {0}; smlBuildTsKv(&kvTs, ts); - - return smlParseEndTelnetJson(info, elements, &kvTs, &kv); + if (info->dataFormat){ + ret = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv); + } else { + ret = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv); + } + return ret; } static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { int32_t payloadNum = 0; int32_t ret = TSDB_CODE_SUCCESS; - if (unlikely(payload == NULL)) { - uError("SML:0x%" PRIx64 " empty JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - info->root = cJSON_Parse(payload); if (unlikely(info->root == NULL)) { uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, payload); @@ -836,13 +830,13 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { return TSDB_CODE_SUCCESS; } -static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *elements) { +static int32_t smlParseJSONString(SSmlHandle *info, char **start, char *end, SSmlLineInfo *elements) { int32_t ret = TSDB_CODE_SUCCESS; if (info->offset[0] == 0) { ret = smlJsonParseObjFirst(start, elements, info->offset); } else { - ret = smlJsonParseObj(start, elements, info->offset); + ret = smlJsonParseObj(start, end, elements, info->offset); } if (ret != TSDB_CODE_SUCCESS) { @@ -870,7 +864,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * elements->cols[elements->colsLen] = tmp; return TSDB_CODE_TSC_INVALID_JSON; } - if (taosArrayPush(info->tagJsonArray, &valueJson) == NULL){ + if (taosArrayPush(info->valueJsonArray, &valueJson) == NULL){ cJSON_Delete(valueJson); elements->cols[elements->colsLen] = tmp; return terrno; @@ -945,20 +939,26 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * SSmlKv kvTs = {0}; smlBuildTsKv(&kvTs, ts); - return smlParseEndTelnetJson(info, elements, &kvTs, &kv); + if (info->dataFormat){ + ret = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv); + } else { + ret = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv); + } + return ret; } int32_t smlParseJSON(SSmlHandle *info, char *payload) { - int32_t payloadNum = 1 << 15; + int32_t payloadNum = 1 << 10; int32_t ret = TSDB_CODE_SUCCESS; uDebug("SML:0x%" PRIx64 "json:%s", info->id, payload); int cnt = 0; char *dataPointStart = payload; + char *dataPointEnd = payload + strlen(payload); while (1) { if (info->dataFormat) { SSmlLineInfo element = {0}; - ret = smlParseJSONString(info, &dataPointStart, &element); + ret = smlParseJSONString(info, &dataPointStart, dataPointEnd, &element); if (element.measureTagsLen != 0) taosMemoryFree(element.measureTag); } else { if (cnt >= payloadNum) { @@ -971,7 +971,7 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { info->lines = (SSmlLineInfo *)tmp; (void)memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); } - ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt); + ret = smlParseJSONString(info, &dataPointStart, dataPointEnd, info->lines + cnt); if ((info->lines + cnt)->measure == NULL) break; } if (unlikely(ret != TSDB_CODE_SUCCESS)) { diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index c1f3431698..5ecf4e2206 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -298,7 +298,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ } if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { - return TSDB_CODE_TSC_INVALID_JSON; + return TSDB_CODE_SML_INVALID_DATA; } cnt++; @@ -311,31 +311,23 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ } static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) { + int32_t code = 0; bool isSameCTable = IS_SAME_CHILD_TABLE; if(isSameCTable){ return TSDB_CODE_SUCCESS; } - int32_t ret = 0; if(info->dataFormat){ - ret = smlProcessSuperTable(info, elements); - if(ret != 0){ - if(info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; - } + SML_CHECK_CODE(smlProcessSuperTable(info, elements)); } - - ret = smlProcessTagLine(info, sql, sqlEnd); - if(ret != 0){ - if (info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; - } - + SML_CHECK_CODE(smlProcessTagLine(info, sql, sqlEnd)); return smlProcessChildTable(info, elements); + +END: + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return code; } static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) { diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index e8601e33bc..bf422675ef 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -148,31 +148,20 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS return TSDB_CODE_SUCCESS; } - int32_t ret = 0; + int32_t code = 0; if(info->dataFormat){ - ret = smlProcessSuperTable(info, elements); - if(ret != 0){ - if(info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; - } - } - - ret = smlProcessTagTelnet(info, data, sqlEnd); - if(ret != 0){ - if (info->reRun){ - return TSDB_CODE_SUCCESS; - } - return ret; - } - - ret = smlJoinMeasureTag(elements); - if(ret != 0){ - return ret; + SML_CHECK_CODE(smlProcessSuperTable(info, elements)); } + SML_CHECK_CODE(smlProcessTagTelnet(info, data, sqlEnd)); + SML_CHECK_CODE(smlJoinMeasureTag(elements)); return smlProcessChildTable(info, elements); + +END: + if(info->reRun){ + return TSDB_CODE_SUCCESS; + } + return code; } // format: =[ =] @@ -239,5 +228,10 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); } - return smlParseEndTelnetJson(info, elements, &kvTs, &kv); + if (info->dataFormat){ + ret = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv); + } else { + ret = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv); + } + return ret; } \ No newline at end of file diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index dc6a302924..bcd12a393a 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -591,6 +591,104 @@ TEST(testCase, smlParseTelnetLine_Test) { // smlDestroyInfo(info); //} +bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->length; + char *endptr = NULL; + double result = taosStr2Double(pVal, &endptr); + if (pVal == endptr) { + smlBuildInvalidDataMsg(msg, "invalid data", pVal); + return false; + } + + int32_t left = len - (endptr - pVal); + if (left == 0 || (left == 3 && strncasecmp(endptr, "f64", left) == 0)) { + kvVal->type = TSDB_DATA_TYPE_DOUBLE; + kvVal->d = result; + } else if ((left == 3 && strncasecmp(endptr, "f32", left) == 0)) { + if (!IS_VALID_FLOAT(result)) { + smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_FLOAT; + kvVal->f = (float)result; + } else if ((left == 1 && *endptr == 'i') || (left == 3 && strncasecmp(endptr, "i64", left) == 0)) { + if (smlDoubleToInt64OverFlow(result)) { + errno = 0; + int64_t tmp = taosStr2Int64(pVal, &endptr, 10); + if (errno == ERANGE) { + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_BIGINT; + kvVal->i = tmp; + return true; + } + kvVal->type = TSDB_DATA_TYPE_BIGINT; + kvVal->i = (int64_t)result; + } else if ((left == 1 && *endptr == 'u') || (left == 3 && strncasecmp(endptr, "u64", left) == 0)) { + if (result >= (double)UINT64_MAX || result < 0) { + errno = 0; + uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); + if (errno == ERANGE || result < 0) { + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_UBIGINT; + kvVal->u = tmp; + return true; + } + kvVal->type = TSDB_DATA_TYPE_UBIGINT; + kvVal->u = result; + } else if (left == 3 && strncasecmp(endptr, "i32", left) == 0) { + if (!IS_VALID_INT(result)) { + smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_INT; + kvVal->i = result; + } else if (left == 3 && strncasecmp(endptr, "u32", left) == 0) { + if (!IS_VALID_UINT(result)) { + smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_UINT; + kvVal->u = result; + } else if (left == 3 && strncasecmp(endptr, "i16", left) == 0) { + if (!IS_VALID_SMALLINT(result)) { + smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_SMALLINT; + kvVal->i = result; + } else if (left == 3 && strncasecmp(endptr, "u16", left) == 0) { + if (!IS_VALID_USMALLINT(result)) { + smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_USMALLINT; + kvVal->u = result; + } else if (left == 2 && strncasecmp(endptr, "i8", left) == 0) { + if (!IS_VALID_TINYINT(result)) { + smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_TINYINT; + kvVal->i = result; + } else if (left == 2 && strncasecmp(endptr, "u8", left) == 0) { + if (!IS_VALID_UTINYINT(result)) { + smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_UTINYINT; + kvVal->u = result; + } else { + smlBuildInvalidDataMsg(msg, "invalid data", pVal); + return false; + } + return true; +} + TEST(testCase, smlParseNumber_performance_Test) { char msg[256] = {0}; SSmlMsgBuf msgBuf; diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index cca35d9c9a..b1ff8cb733 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -468,35 +468,38 @@ end: int32_t smlInitHandle(SQuery** query) { *query = NULL; SQuery* pQuery = NULL; + SVnodeModifyOpStmt* stmt = NULL; + int32_t code = nodesMakeNode(QUERY_NODE_QUERY, (SNode**)&pQuery); - if (NULL == pQuery) { + if (code != 0) { uError("create pQuery error"); - return code; + goto END; } pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->haveResultSet = false; pQuery->msgType = TDMT_VND_SUBMIT; - SVnodeModifyOpStmt* stmt = NULL; code = nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT, (SNode**)&stmt); - if (NULL == stmt) { + if (code != 0) { uError("create SVnodeModifyOpStmt error"); - qDestroyQuery(pQuery); - return code; + goto END; } stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (stmt->pTableBlockHashObj == NULL){ uError("create pTableBlockHashObj error"); - qDestroyQuery(pQuery); - nodesDestroyNode((SNode*)stmt); - return terrno; + code = terrno; + goto END; } stmt->freeHashFunc = insDestroyTableDataCxtHashMap; stmt->freeArrayFunc = insDestroyVgroupDataCxtList; pQuery->pRoot = (SNode*)stmt; *query = pQuery; + return code; - return TSDB_CODE_SUCCESS; +END: + nodesDestroyNode((SNode*)stmt); + qDestroyQuery(pQuery); + return code; } int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) { From 2e8a5348c9c9438f4341e5838104f420b8ceef5e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 22 Oct 2024 17:32:18 +0800 Subject: [PATCH 06/34] enh:[TD-32166]refactor code in sml --- source/client/inc/clientSml.h | 24 ++++++--------- source/client/src/clientSml.c | 58 ++++++++++++++--------------------- 2 files changed, 33 insertions(+), 49 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 3098b124b7..88831ce815 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -92,21 +92,17 @@ extern "C" { } \ } -#define SML_CHECK_CODE(CMD) \ - do { \ - code = (CMD); \ - if (TSDB_CODE_SUCCESS != code) { \ - goto END; \ - } \ - } while (0) +#define SML_CHECK_CODE(CMD) \ + code = (CMD); \ + if (TSDB_CODE_SUCCESS != code) { \ + goto END; \ + } -#define SML_CHECK_NULL(CMD) \ - do { \ - if (NULL == (CMD)) { \ - code = terrno; \ - goto END; \ - } \ - } while (0) +#define SML_CHECK_NULL(CMD) \ + if (NULL == (CMD)) { \ + code = terrno; \ + goto END; \ + } typedef enum { SCHEMA_ACTION_NULL, diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 25abeb52fb..c5842bb0bb 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -207,19 +207,18 @@ static void smlDestroySTableMeta(void *para) { } int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta **sMeta) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + STableMeta *pTableMeta = NULL; + char *measure = currElement->measure; int measureLen = currElement->measureLen; if (currElement->measureEscaped) { measure = (char *)taosMemoryMalloc(measureLen); - if (measure == NULL) { - return terrno; - } + SML_CHECK_NULL(measure); (void)memcpy(measure, currElement->measure, measureLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); smlStrReplace(measure, measureLen); } - STableMeta *pTableMeta = NULL; code = smlGetMeta(info, measure, measureLen, &pTableMeta); if (currElement->measureEscaped) { taosMemoryFree(measure); @@ -227,20 +226,9 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml if (code != TSDB_CODE_SUCCESS) { info->dataFormat = false; info->reRun = true; - taosMemoryFreeClear(pTableMeta); - return code; - } - code = smlBuildSTableMeta(info->dataFormat, sMeta); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pTableMeta); - return code; - } - (*sMeta)->tableMeta = pTableMeta; - code = taosHashPut(info->superTables, currElement->measure, currElement->measureLen, sMeta, POINTER_BYTES); - if (code != TSDB_CODE_SUCCESS) { - smlDestroySTableMeta(*sMeta); - return code; + goto END; } + SML_CHECK_CODE(smlBuildSTableMeta(info->dataFormat, sMeta)); for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { SSchema *col = pTableMeta->schema + i; SSmlKv kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type}; @@ -254,16 +242,19 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml } if (i < pTableMeta->tableInfo.numOfColumns) { - if (taosArrayPush((*sMeta)->cols, &kv) == NULL) { - return terrno; - } + SML_CHECK_NULL(taosArrayPush((*sMeta)->cols, &kv)); } else { - if (taosArrayPush((*sMeta)->tags, &kv) == NULL) { - return terrno; - } + SML_CHECK_NULL(taosArrayPush((*sMeta)->tags, &kv)); } } - return TSDB_CODE_SUCCESS; + SML_CHECK_CODE(taosHashPut(info->superTables, currElement->measure, currElement->measureLen, sMeta, POINTER_BYTES)); + (*sMeta)->tableMeta = pTableMeta; + return code; + +END: + smlDestroySTableMeta(*sMeta); + taosMemoryFreeClear(pTableMeta); + return code; } bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { @@ -863,7 +854,7 @@ static FORCE_INLINE void smlBuildCreateStbReq(SMCreateStbReq *pReq, int32_t colV pReq->suid = suid; pReq->source = source; } -static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta, +static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray **pTags, STableMeta *pTableMeta, ESchemaAction action) { SRequestObj *pRequest = NULL; SMCreateStbReq pReq = {0}; @@ -873,8 +864,9 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, // put front for free pReq.numOfColumns = taosArrayGetSize(pColumns); - pReq.pTags = pTags; - pReq.numOfTags = taosArrayGetSize(pTags); + pReq.pTags = *pTags; + pReq.numOfTags = taosArrayGetSize(*pTags); + *pTags = NULL; pReq.pColumns = taosArrayInit(pReq.numOfColumns, sizeof(SFieldWithOptions)); SML_CHECK_NULL(pReq.pColumns); @@ -970,8 +962,7 @@ static int32_t smlCreateTable(SSmlHandle *info, SRequestConnInfo *conn, SSmlSTab SML_CHECK_NULL(pTags); SML_CHECK_CODE(smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true)); SML_CHECK_CODE(smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false)); - pTags = NULL; - SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE)); + SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, &pTags, NULL, SCHEMA_ACTION_CREATE_STABLE)); info->cost.numOfCreateSTables++; taosMemoryFreeClear(*pTableMeta); @@ -1019,8 +1010,7 @@ static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInf SML_CHECK_CODE(smlBuildFieldsList(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, pTags, (*pTableMeta)->tableInfo.numOfColumns, true)); - pTags = NULL; - SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, pTags, (*pTableMeta), action)); + SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, &pTags, (*pTableMeta), action)); info->cost.numOfAlterTagSTables++; taosMemoryFreeClear(*pTableMeta); @@ -1050,8 +1040,7 @@ static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmp, SRequestConnIn SML_CHECK_CODE(smlBuildFieldsList(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, pColumns, (*pTableMeta)->tableInfo.numOfColumns, false)); - pTags = NULL; - SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, pTags, (*pTableMeta), action)); + SML_CHECK_CODE(smlSendMetaMsg(info, pName, pColumns, &pTags, (*pTableMeta), action)); info->cost.numOfAlterColSTables++; taosMemoryFreeClear(*pTableMeta); @@ -1409,7 +1398,6 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { info->lineNum); SSmlSTableMeta *meta = NULL; SML_CHECK_CODE(smlBuildSTableMeta(info->dataFormat, &meta)); - SML_CHECK_CODE(taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES)); code = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); if (code != TSDB_CODE_SUCCESS) { smlDestroySTableMeta(meta); From 21a7b6dc16d0d4cdf60d348e86330723c723d3ed Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 22 Oct 2024 17:43:30 +0800 Subject: [PATCH 07/34] enh:[TD-32166]remove useless code in sml --- source/client/inc/clientSml.h | 4 +- source/client/src/clientSml.c | 4 +- source/client/src/clientSmlJson.c | 422 +----------------------------- 3 files changed, 4 insertions(+), 426 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 88831ce815..a562ca3226 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -225,8 +225,6 @@ extern int64_t smlFactorS[]; int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle); void smlDestroyInfo(SSmlHandle *info); -int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); -int smlJsonParseObj(char **start, char *end, SSmlLineInfo *element, int8_t *offset); void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); @@ -246,7 +244,7 @@ void smlDestroyTableInfo(void *para); void freeSSmlKv(void* data); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); -int32_t smlParseJSON(SSmlHandle *info, char *payload); +int32_t smlParseJSONExt(SSmlHandle *info, char *payload); int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta** sMeta); bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index c5842bb0bb..9f9671f340 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1597,9 +1597,9 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char int32_t code = TSDB_CODE_SUCCESS; if (info->protocol == TSDB_SML_JSON_PROTOCOL) { if (lines) { - code = smlParseJSON(info, *lines); + code = smlParseJSONExt(info, *lines); } else if (rawLine) { - code = smlParseJSON(info, rawLine); + code = smlParseJSONExt(info, rawLine); } if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlParseJSON failed:%s", info->id, lines ? *lines : rawLine); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 9568074018..d44f9cac9c 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -21,259 +21,6 @@ #define OTD_JSON_SUB_FIELDS_NUM 2 -#define JUMP_JSON_SPACE(start) \ - while (*(start)) { \ - if (unlikely(*(start) > 32)) \ - break; \ - else \ - (start)++; \ - } - -static int32_t smlJsonGetObj(char **payload) { - int leftBracketCnt = 0; - bool isInQuote = false; - while (**payload) { - if (**payload == '"' && *((*payload) - 1) != '\\') { - isInQuote = !isInQuote; - } else if (!isInQuote && unlikely(**payload == '{')) { - leftBracketCnt++; - (*payload)++; - continue; - } else if (!isInQuote && unlikely(**payload == '}')) { - leftBracketCnt--; - (*payload)++; - if (leftBracketCnt == 0) { - return 0; - } else if (leftBracketCnt < 0) { - return -1; - } - continue; - } - (*payload)++; - } - return -1; -} - -int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { - int index = 0; - while (*(*start)) { - if ((*start)[0] != '"') { - (*start)++; - continue; - } - - if (unlikely(index >= OTD_JSON_FIELDS_NUM)) { - uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start); - return TSDB_CODE_TSC_INVALID_JSON; - } - - char *sTmp = *start; - if ((*start)[1] == 'm' && (*start)[2] == 'e' && (*start)[3] == 't' && (*start)[4] == 'r' && (*start)[5] == 'i' && - (*start)[6] == 'c' && (*start)[7] == '"') { - (*start) += 8; - bool isInQuote = false; - while (*(*start)) { - if (unlikely(!isInQuote && *(*start) == '"')) { - (*start)++; - offset[index++] = *start - sTmp; - element->measure = (*start); - isInQuote = true; - continue; - } - if (unlikely(isInQuote && *(*start) == '"')) { - element->measureLen = (*start) - element->measure; - (*start)++; - break; - } - (*start)++; - } - } else if ((*start)[1] == 't' && (*start)[2] == 'i' && (*start)[3] == 'm' && (*start)[4] == 'e' && - (*start)[5] == 's' && (*start)[6] == 't' && (*start)[7] == 'a' && (*start)[8] == 'm' && - (*start)[9] == 'p' && (*start)[10] == '"') { - (*start) += 11; - bool hasColon = false; - while (*(*start)) { - if (unlikely(!hasColon && *(*start) == ':')) { - (*start)++; - JUMP_JSON_SPACE((*start)) - offset[index++] = *start - sTmp; - element->timestamp = (*start); - if (*(*start) == '{') { - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->timestampLen = tmp - (*start); - *start = tmp; - } - break; - } - hasColon = true; - continue; - } - if (unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))) { - element->timestampLen = (*start) - element->timestamp; - break; - } - (*start)++; - } - } else if ((*start)[1] == 'v' && (*start)[2] == 'a' && (*start)[3] == 'l' && (*start)[4] == 'u' && - (*start)[5] == 'e' && (*start)[6] == '"') { - (*start) += 7; - - bool hasColon = false; - while (*(*start)) { - if (unlikely(!hasColon && *(*start) == ':')) { - (*start)++; - JUMP_JSON_SPACE((*start)) - offset[index++] = *start - sTmp; - element->cols = (*start); - if (*(*start) == '{') { - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->colsLen = tmp - (*start); - *start = tmp; - } - break; - } - hasColon = true; - continue; - } - if (unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))) { - element->colsLen = (*start) - element->cols; - break; - } - (*start)++; - } - } else if ((*start)[1] == 't' && (*start)[2] == 'a' && (*start)[3] == 'g' && (*start)[4] == 's' && - (*start)[5] == '"') { - (*start) += 6; - - while (*(*start)) { - if (unlikely(*(*start) == ':')) { - (*start)++; - JUMP_JSON_SPACE((*start)) - offset[index++] = *start - sTmp; - element->tags = (*start); - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->tagsLen = tmp - (*start); - *start = tmp; - } - break; - } - (*start)++; - } - } - if (*(*start) == '\0') { - break; - } - if (*(*start) == '}') { - (*start)++; - break; - } - (*start)++; - } - - if (unlikely(index != OTD_JSON_FIELDS_NUM) || element->tags == NULL || element->cols == NULL || - element->measure == NULL || element->timestamp == NULL) { - uError("elements != %d or element parse null", OTD_JSON_FIELDS_NUM); - return TSDB_CODE_TSC_INVALID_JSON; - } - return 0; -} - -int smlJsonParseObj(char **start, char *end, SSmlLineInfo *element, int8_t *offset) { - int index = 0; - while (*(*start)) { - if ((*start)[0] != '"') { - (*start)++; - continue; - } - - if (unlikely(index >= OTD_JSON_FIELDS_NUM)) { - uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start); - return TSDB_CODE_TSC_INVALID_JSON; - } - - if ((*start) + offset[index] >= end){ - return TSDB_CODE_TSC_INVALID_JSON; - } - - if ((*start)[1] == 'm') { - (*start) += offset[index++]; - element->measure = *start; - while (*(*start)) { - if (unlikely(*(*start) == '"')) { - element->measureLen = (*start) - element->measure; - (*start)++; - break; - } - (*start)++; - } - } else if ((*start)[1] == 't' && (*start)[2] == 'i') { - (*start) += offset[index++]; - element->timestamp = *start; - if (*(*start) == '{') { - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->timestampLen = tmp - (*start); - *start = tmp; - } - } else { - while (*(*start)) { - if (unlikely(*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)) { - element->timestampLen = (*start) - element->timestamp; - break; - } - (*start)++; - } - } - } else if ((*start)[1] == 'v') { - (*start) += offset[index++]; - element->cols = *start; - if (*(*start) == '{') { - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->colsLen = tmp - (*start); - *start = tmp; - } - } else { - while (*(*start)) { - if (unlikely(*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)) { - element->colsLen = (*start) - element->cols; - break; - } - (*start)++; - } - } - } else if ((*start)[1] == 't' && (*start)[2] == 'a') { - (*start) += offset[index++]; - element->tags = (*start); - char *tmp = *start; - int32_t code = smlJsonGetObj(&tmp); - if (code == 0) { - element->tagsLen = tmp - (*start); - *start = tmp; - } - } - if (*(*start) == '}') { - (*start)++; - break; - } - (*start)++; - } - - if (unlikely(index != 0 && index != OTD_JSON_FIELDS_NUM)) { - uError("elements != %d", OTD_JSON_FIELDS_NUM); - return TSDB_CODE_TSC_INVALID_JSON; - } - return TSDB_CODE_SUCCESS; -} - static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { elements->measureLen = strlen(metric->valuestring); if (IS_INVALID_TABLE_LEN(elements->measureLen)) { @@ -760,7 +507,7 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo return ret; } -static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { +int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { int32_t payloadNum = 0; int32_t ret = TSDB_CODE_SUCCESS; @@ -830,170 +577,3 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { return TSDB_CODE_SUCCESS; } -static int32_t smlParseJSONString(SSmlHandle *info, char **start, char *end, SSmlLineInfo *elements) { - int32_t ret = TSDB_CODE_SUCCESS; - - if (info->offset[0] == 0) { - ret = smlJsonParseObjFirst(start, elements, info->offset); - } else { - ret = smlJsonParseObj(start, end, elements, info->offset); - } - - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - if (unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS; - - if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) { - smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - - SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; - - if (unlikely(elements->colsLen == 0)) { - uError("SML:colsLen == 0"); - return TSDB_CODE_TSC_INVALID_VALUE; - } else if (unlikely(elements->cols[0] == '{')) { - char tmp = elements->cols[elements->colsLen]; - elements->cols[elements->colsLen] = '\0'; - cJSON *valueJson = cJSON_Parse(elements->cols); - if (unlikely(valueJson == NULL)) { - uError("SML:0x%" PRIx64 " parse json cols failed:%s", info->id, elements->cols); - elements->cols[elements->colsLen] = tmp; - return TSDB_CODE_TSC_INVALID_JSON; - } - if (taosArrayPush(info->valueJsonArray, &valueJson) == NULL){ - cJSON_Delete(valueJson); - elements->cols[elements->colsLen] = tmp; - return terrno; - } - ret = smlParseValueFromJSONObj(valueJson, &kv); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " Failed to parse value from JSON Obj:%s", info->id, elements->cols); - elements->cols[elements->colsLen] = tmp; - return TSDB_CODE_TSC_INVALID_VALUE; - } - elements->cols[elements->colsLen] = tmp; - } else if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " cols invalidate:%s", info->id, elements->cols); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - // Parse tags - if (is_same_child_table_telnet(elements, &info->preLine) != 0) { - char tmp = *(elements->tags + elements->tagsLen); - *(elements->tags + elements->tagsLen) = 0; - cJSON *tagsJson = cJSON_Parse(elements->tags); - *(elements->tags + elements->tagsLen) = tmp; - if (unlikely(tagsJson == NULL)) { - uError("SML:0x%" PRIx64 " parse json tag failed:%s", info->id, elements->tags); - return TSDB_CODE_TSC_INVALID_JSON; - } - - if (taosArrayPush(info->tagJsonArray, &tagsJson) == NULL){ - cJSON_Delete(tagsJson); - uError("SML:0x%" PRIx64 " taosArrayPush failed", info->id); - return terrno; - } - ret = smlParseTagsFromJSON(info, tagsJson, elements); - if (unlikely(ret)) { - uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); - return ret; - } - } else { - elements->measureTag = info->preLine.measureTag; - } - - if (unlikely(info->reRun)) { - return TSDB_CODE_SUCCESS; - } - - // Parse timestamp - // notice!!! put ts back to tag to ensure get meta->precision - int64_t ts = 0; - if (unlikely(elements->timestampLen == 0)) { - uError("OTD:0x%" PRIx64 " elements->timestampLen == 0", info->id); - return TSDB_CODE_INVALID_TIMESTAMP; - } else if (elements->timestamp[0] == '{') { - char tmp = elements->timestamp[elements->timestampLen]; - elements->timestamp[elements->timestampLen] = '\0'; - cJSON *tsJson = cJSON_Parse(elements->timestamp); - ts = smlParseTSFromJSON(info, tsJson); - if (unlikely(ts < 0)) { - uError("SML:0x%" PRIx64 " Unable to parse timestamp from JSON payload:%s", info->id, elements->timestamp); - elements->timestamp[elements->timestampLen] = tmp; - cJSON_Delete(tsJson); - return TSDB_CODE_INVALID_TIMESTAMP; - } - elements->timestamp[elements->timestampLen] = tmp; - cJSON_Delete(tsJson); - } else { - ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen); - if (unlikely(ts < 0)) { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); - return TSDB_CODE_INVALID_TIMESTAMP; - } - } - SSmlKv kvTs = {0}; - smlBuildTsKv(&kvTs, ts); - - if (info->dataFormat){ - ret = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv); - } else { - ret = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv); - } - return ret; -} - -int32_t smlParseJSON(SSmlHandle *info, char *payload) { - int32_t payloadNum = 1 << 10; - int32_t ret = TSDB_CODE_SUCCESS; - - uDebug("SML:0x%" PRIx64 "json:%s", info->id, payload); - int cnt = 0; - char *dataPointStart = payload; - char *dataPointEnd = payload + strlen(payload); - while (1) { - if (info->dataFormat) { - SSmlLineInfo element = {0}; - ret = smlParseJSONString(info, &dataPointStart, dataPointEnd, &element); - if (element.measureTagsLen != 0) taosMemoryFree(element.measureTag); - } else { - if (cnt >= payloadNum) { - payloadNum = payloadNum << 1; - void *tmp = taosMemoryRealloc(info->lines, payloadNum * sizeof(SSmlLineInfo)); - if (tmp == NULL) { - ret = terrno; - return ret; - } - info->lines = (SSmlLineInfo *)tmp; - (void)memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); - } - ret = smlParseJSONString(info, &dataPointStart, dataPointEnd, info->lines + cnt); - if ((info->lines + cnt)->measure == NULL) break; - } - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - uError("SML:0x%" PRIx64 " Invalid JSON Payload 1:%s", info->id, payload); - return smlParseJSONExt(info, payload); - } - - if (unlikely(info->reRun)) { - cnt = 0; - dataPointStart = payload; - info->lineNum = payloadNum; - ret = smlClearForRerun(info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - continue; - } - - cnt++; - if (*dataPointStart == '\0') break; - } - info->lineNum = cnt; - - return TSDB_CODE_SUCCESS; -} From 574998f02835372490b6d5b9cfeec555a34ebb78 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 22 Oct 2024 17:50:26 +0800 Subject: [PATCH 08/34] enh:[TD-32166]remove useless code in sml --- source/client/src/clientSml.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 9f9671f340..c17ff1413e 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -197,7 +197,13 @@ void smlBuildTsKv(SSmlKv *kv, int64_t ts) { } static void smlDestroySTableMeta(void *para) { + if (para == NULL) { + return; + } SSmlSTableMeta *meta = *(SSmlSTableMeta **)para; + if (meta == NULL) { + return; + } taosHashCleanup(meta->tagHash); taosHashCleanup(meta->colHash); taosArrayDestroy(meta->tags); @@ -252,7 +258,7 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml return code; END: - smlDestroySTableMeta(*sMeta); + smlDestroySTableMeta(sMeta); taosMemoryFreeClear(pTableMeta); return code; } @@ -587,7 +593,7 @@ int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta **sMeta) { return TSDB_CODE_SUCCESS; END: - smlDestroySTableMeta(meta); + smlDestroySTableMeta(&meta); return TSDB_CODE_OUT_OF_MEMORY; } @@ -1400,7 +1406,7 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { SML_CHECK_CODE(smlBuildSTableMeta(info->dataFormat, &meta)); code = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); if (code != TSDB_CODE_SUCCESS) { - smlDestroySTableMeta(meta); + smlDestroySTableMeta(&meta); uError("SML:0x%" PRIx64 " put measuer to hash failed", info->id); goto END; } From 75c8727c52909818b0ed5ac3a25ecced9b9a508b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 24 Oct 2024 15:48:57 +0800 Subject: [PATCH 09/34] enh:[TD-32166]refactor code in sml --- source/client/inc/clientSml.h | 8 ++ source/client/src/clientSml.c | 167 ++++++++++++++-------------- source/client/src/clientSmlJson.c | 89 +++++---------- source/client/src/clientSmlLine.c | 3 +- source/client/src/clientSmlTelnet.c | 7 +- utils/test/c/sml_test.c | 104 ++++++++++++++++- 6 files changed, 230 insertions(+), 148 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index a562ca3226..8558ec46dc 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -95,15 +95,23 @@ extern "C" { #define SML_CHECK_CODE(CMD) \ code = (CMD); \ if (TSDB_CODE_SUCCESS != code) { \ + lino = __LINE__; \ goto END; \ } #define SML_CHECK_NULL(CMD) \ if (NULL == (CMD)) { \ code = terrno; \ + lino = __LINE__; \ goto END; \ } +#define RETURN \ + if (code != 0){ \ + uError("%s failed code:%d line:%d", __FUNCTION__ , code, lino); \ + } \ + return code; + typedef enum { SCHEMA_ACTION_NULL, SCHEMA_ACTION_CREATE_STABLE, diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index c17ff1413e..5f4327b1cd 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -213,22 +213,20 @@ static void smlDestroySTableMeta(void *para) { } int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta **sMeta) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; STableMeta *pTableMeta = NULL; - char *measure = currElement->measure; - int measureLen = currElement->measureLen; + int measureLen = currElement->measureLen; + char *measure = (char *)taosMemoryMalloc(measureLen); + SML_CHECK_NULL(measure); + (void)memcpy(measure, currElement->measure, measureLen); if (currElement->measureEscaped) { - measure = (char *)taosMemoryMalloc(measureLen); - SML_CHECK_NULL(measure); - (void)memcpy(measure, currElement->measure, measureLen); PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); - smlStrReplace(measure, measureLen); } + smlStrReplace(measure, measureLen); code = smlGetMeta(info, measure, measureLen, &pTableMeta); - if (currElement->measureEscaped) { - taosMemoryFree(measure); - } + taosMemoryFree(measure); if (code != TSDB_CODE_SUCCESS) { info->dataFormat = false; info->reRun = true; @@ -260,7 +258,7 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml END: smlDestroySTableMeta(sMeta); taosMemoryFreeClear(pTableMeta); - return code; + RETURN } bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { @@ -376,8 +374,8 @@ int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) { int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { int32_t code = TSDB_CODE_SUCCESS; - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen); + int32_t lino = 0; + SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen); SSmlTableInfo *tinfo = NULL; if (unlikely(oneTable == NULL)) { SML_CHECK_CODE(smlBuildTableInfo(1, elements->measure, elements->measureLen, &tinfo)); @@ -401,47 +399,40 @@ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { } else { tinfo = *oneTable; } - if (tinfo == NULL) { - uError("smlProcessChildTable failed to get child table info"); - return TSDB_CODE_SML_INTERNAL_ERROR; - } if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; return TSDB_CODE_SUCCESS; END: smlDestroyTableInfo(&tinfo); - return code; + RETURN } int32_t smlParseEndTelnetJsonFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv) { int32_t code = 0; - uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format true, ts:%" PRId64, info->id, kvTs->i); + int32_t lino = 0; + uDebug("SML:0x%" PRIx64 " %s format true, ts:%" PRId64, info->id, __FUNCTION__ , kvTs->i); SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0)); SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1)); SML_CHECK_CODE(smlBuildRow(info->currTableDataCtx)); - info->preLine = *elements; END: clearColValArraySml(info->currTableDataCtx->pValues); - if (unlikely(code != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); - } - return code; + RETURN } int32_t smlParseEndTelnetJsonUnFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv) { int32_t code = 0; - uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i); + int32_t lino = 0; + uDebug("SML:0x%" PRIx64 " %s format false, ts:%" PRId64, info->id, __FUNCTION__, kvTs->i); if (elements->colArray == NULL) { elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); SML_CHECK_NULL(elements->colArray); } SML_CHECK_NULL(taosArrayPush(elements->colArray, kvTs)); SML_CHECK_NULL (taosArrayPush(elements->colArray, kv)); - info->preLine = *elements; END: - return code; + RETURN } int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) { @@ -469,8 +460,9 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnameKey) { int32_t code = 0; - bool autoChildName = false; - size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter); + int32_t lino = 0; + bool autoChildName = false; + size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter); if (delimiter > 0 && tbnameKey == NULL) { size_t totalNameLen = delimiter * (taosArrayGetSize(tags) - 1); for (int i = 0; i < taosArrayGetSize(tags); i++) { @@ -519,11 +511,12 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam } END: - return code; + RETURN } int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { int32_t code = 0; + int32_t lino = 0; SArray *dst = NULL; SML_CHECK_CODE(smlParseTableName(oneTable->tags, oneTable->childTableName, tbnameKey)); @@ -531,7 +524,6 @@ int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { dst = taosArrayDup(oneTable->tags, NULL); SML_CHECK_NULL(dst); if (oneTable->sTableNameLen >= TSDB_TABLE_NAME_LEN) { - uError("SML:smlSetCTableName super table name is too long"); code = TSDB_CODE_SML_INTERNAL_ERROR; goto END; } @@ -550,7 +542,7 @@ int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { END: taosArrayDestroy(dst); - return code; + RETURN } int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo) { @@ -575,6 +567,7 @@ int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo * int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta **sMeta) { int32_t code = 0; + int32_t lino = 0; SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); SML_CHECK_NULL(meta); if (unlikely(!isDataFormat)) { @@ -594,7 +587,8 @@ int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta **sMeta) { END: smlDestroySTableMeta(&meta); - return TSDB_CODE_OUT_OF_MEMORY; + uError("%s failed code:%d line:%d", __FUNCTION__ , code, lino); + return code; } int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { @@ -704,8 +698,9 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; if (index) { if (colField[*index].type != kv->type) { - uError("SML:0x%" PRIx64 " point type and db type mismatch. db type: %d, point type: %d, key: %s", info->id, - colField[*index].type, kv->type, kv->key); + snprintf(info->msgBuf.buf, info->msgBuf.len, "SML:0x%" PRIx64 " %s point type and db type mismatch. db type: %d, point type: %d, key: %s", + info->id, __FUNCTION__, colField[*index].type, kv->type, kv->key); + uError("%s", info->msgBuf.buf); return TSDB_CODE_SML_INVALID_DATA; } @@ -759,6 +754,7 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray *checkDumplicateCols, ESchemaAction *action, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; for (int j = 0; j < taosArrayGetSize(cols); ++j) { if (j == 0 && !isTag) continue; SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); @@ -775,11 +771,12 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH } } END: - return code; + RETURN } static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); SML_CHECK_NULL(hashTmp); int32_t i = 0; @@ -798,7 +795,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool END: taosHashCleanup(hashTmp); - return code; + RETURN } static int32_t getBytes(uint8_t type, int32_t length) { @@ -813,6 +810,7 @@ static int32_t getBytes(uint8_t type, int32_t length) { static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray *results, int32_t numOfCols, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = TSDB_CODE_SUCCESS; for (int j = 0; j < taosArrayGetSize(cols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); SML_CHECK_NULL(kv); @@ -822,7 +820,7 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO SField field = {0}; field.type = kv->type; field.bytes = getBytes(kv->type, kv->length); - (void)memcpy(field.name, kv->key, MIN(kv->keyLen, sizeof(field.name) - 1)); + (void)memcpy(field.name, kv->key, TMIN(kv->keyLen, sizeof(field.name) - 1)); SML_CHECK_NULL(taosArrayPush(results, &field)); } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); @@ -851,7 +849,7 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO } END: - return code; + RETURN } static FORCE_INLINE void smlBuildCreateStbReq(SMCreateStbReq *pReq, int32_t colVer, int32_t tagVer, tb_uid_t suid, int8_t source){ @@ -865,6 +863,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SRequestObj *pRequest = NULL; SMCreateStbReq pReq = {0}; int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SCmdMsgInfo pCmdMsg = {0}; char *pSql = NULL; @@ -952,16 +951,17 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, END: destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); - return code; + RETURN } static int32_t smlCreateTable(SSmlHandle *info, SRequestConnInfo *conn, SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){ int32_t code = 0; + int32_t lino = 0; SArray *pColumns = NULL; SArray *pTags = NULL; SML_CHECK_CODE(smlCheckAuth(info, conn, NULL, AUTH_TYPE_WRITE)); - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName->tname); + uDebug("SML:0x%" PRIx64 " %s create table:%s", info->id, __FUNCTION__, pName->tname); pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); SML_CHECK_NULL(pColumns); pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); @@ -977,11 +977,12 @@ static int32_t smlCreateTable(SSmlHandle *info, SRequestConnInfo *conn, SSmlSTab END: taosArrayDestroy(pColumns); taosArrayDestroy(pTags); - return code; + RETURN } static int32_t smlBuildFields(SArray **pColumns, SArray **pTags, STableMeta *pTableMeta, SSmlSTableMeta *sTableData){ int32_t code = 0; + int32_t lino = 0; *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + (pTableMeta)->tableInfo.numOfColumns, sizeof(SField)); SML_CHECK_NULL(pColumns); *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + (pTableMeta)->tableInfo.numOfTags, sizeof(SField)); @@ -998,7 +999,7 @@ static int32_t smlBuildFields(SArray **pColumns, SArray **pTags, STableMeta *pTa } } END: - return code; + RETURN } static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn, SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){ @@ -1006,11 +1007,12 @@ static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInf SArray *pColumns = NULL; SArray *pTags = NULL; int32_t code = 0; + int32_t lino = 0; SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, sTableData->cols, &action, true)); if (action != SCHEMA_ACTION_NULL) { SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE)); - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName->tname, + uDebug("SML:0x%" PRIx64 " %s change table tag, table:%s, action:%d", info->id, __FUNCTION__, pName->tname, action); SML_CHECK_CODE(smlBuildFields(&pColumns, &pTags, *pTableMeta, sTableData)); SML_CHECK_CODE(smlBuildFieldsList(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, pTags, @@ -1027,7 +1029,7 @@ static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInf END: taosArrayDestroy(pColumns); taosArrayDestroy(pTags); - return code; + RETURN } static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn, @@ -1036,11 +1038,12 @@ static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmp, SRequestConnIn SArray *pColumns = NULL; SArray *pTags = NULL; int32_t code = 0; + int32_t lino = 0; SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false)); if (action != SCHEMA_ACTION_NULL) { SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE)); - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName->tname, + uDebug("SML:0x%" PRIx64 " %s change table col, table:%s, action:%d", info->id, __FUNCTION__, pName->tname, action); SML_CHECK_CODE(smlBuildFields(&pColumns, &pTags, *pTableMeta, sTableData)); SML_CHECK_CODE(smlBuildFieldsList(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, pColumns, @@ -1054,14 +1057,15 @@ static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmp, SRequestConnIn SML_CHECK_CODE(catalogGetSTableMeta(info->pCatalog, conn, pName, pTableMeta)); } - END: +END: taosArrayDestroy(pColumns); taosArrayDestroy(pTags); - return code; + RETURN } static int32_t smlBuildTempHash(SHashObj *hashTmp, STableMeta *pTableMeta, uint16_t start, uint16_t end){ int32_t code = 0; + int32_t lino = 0; for (uint16_t i = start; i < end; i++) { SML_CHECK_CODE(taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES)); } @@ -1071,12 +1075,13 @@ END: } static int32_t smlModifyDBSchemas(SSmlHandle *info) { - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas start, format:%d, needModifySchema:%d", info->id, info->dataFormat, + uDebug("SML:0x%" PRIx64 " %s start, format:%d, needModifySchema:%d", info->id, __FUNCTION__, info->dataFormat, info->needModifySchema); if (info->dataFormat && !info->needModifySchema) { return TSDB_CODE_SUCCESS; } int32_t code = 0; + int32_t lino = 0; SHashObj *hashTmp = NULL; STableMeta *pTableMeta = NULL; @@ -1099,10 +1104,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { char *measure = taosMemoryMalloc(superTableLen); SML_CHECK_NULL(measure); (void)memcpy(measure, superTable, superTableLen); - PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); + if (info->protocol == TSDB_SML_LINE_PROTOCOL){ + PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); + } smlStrReplace(measure, superTableLen); (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - (void)memcpy(pName.tname, measure, MIN(superTableLen, TSDB_TABLE_NAME_LEN - 1)); + (void)memcpy(pName.tname, measure, TMIN(superTableLen, TSDB_TABLE_NAME_LEN - 1)); taosMemoryFree(measure); code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); @@ -1160,6 +1167,7 @@ END: static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SHashObj *checkDuplicate) { int32_t code = 0; + int32_t lino = 0; terrno = 0; for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); @@ -1177,12 +1185,13 @@ static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols } END: - return code; + RETURN } static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg, SHashObj *checkDuplicate) { int32_t code = 0; + int32_t lino = 0; for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); SML_CHECK_NULL(kv); @@ -1225,7 +1234,7 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols } END: - return code; + RETURN } void smlDestroyTableInfo(void *para) { @@ -1249,8 +1258,7 @@ void freeSSmlKv(void *data) { } void smlDestroyInfo(SSmlHandle *info) { - if (!info) return; - // qDestroyQuery(info->pQuery); + if (info == NULL) return; taosHashCleanup(info->pVgHash); taosHashCleanup(info->childTables); @@ -1275,22 +1283,20 @@ void smlDestroyInfo(SSmlHandle *info) { if (!info->dataFormat) { for (int i = 0; i < info->lineNum; i++) { taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv); - if (info->parseJsonByLib) { - taosMemoryFree(info->lines[i].tags); - } if (info->lines[i].measureTagsLen != 0 && info->protocol != TSDB_SML_LINE_PROTOCOL) { taosMemoryFree(info->lines[i].measureTag); } } taosMemoryFree(info->lines); } - + taosMemoryFreeClear(info->preLine.tags); cJSON_Delete(info->root); taosMemoryFreeClear(info); } int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); SML_CHECK_NULL(info); if (taos != NULL){ @@ -1327,11 +1333,12 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { END: smlDestroyInfo(info); - return code; + RETURN } static int32_t smlPushCols(SArray *colsArray, SArray *cols) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); SML_CHECK_NULL(kvHash); for (size_t i = 0; i < taosArrayGetSize(cols); i++) { @@ -1350,13 +1357,14 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { return code; END: taosHashCleanup(kvHash); - return code; + RETURN } static int32_t smlParseLineBottom(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlParseLineBottom start, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); int32_t code = 0; + int32_t lino = 0; if (info->dataFormat) return TSDB_CODE_SUCCESS; for (int32_t i = 0; i < info->lineNum; i++) { @@ -1417,11 +1425,12 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlParseLineBottom end, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); END: - return code; + RETURN } static int32_t smlInsertData(SSmlHandle *info) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; char *measure = NULL; SSmlTableInfo **oneTable = NULL; uDebug("SML:0x%" PRIx64 " smlInsertData start, format:%d", info->id, info->dataFormat); @@ -1444,7 +1453,9 @@ static int32_t smlInsertData(SSmlHandle *info) { measure = (char *)taosMemoryMalloc(tableData->sTableNameLen); SML_CHECK_NULL(measure); (void)memcpy(measure, tableData->sTableName, tableData->sTableNameLen); - PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + if (info->protocol == TSDB_SML_LINE_PROTOCOL){ + PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + } smlStrReplace(measure, measureLen); (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); (void)memcpy(pName.tname, measure, measureLen); @@ -1505,7 +1516,7 @@ static int32_t smlInsertData(SSmlHandle *info) { END: taosMemoryFree(measure); taosHashCancelIterate(info->childTables, oneTable); - return code; + RETURN } static void smlPrintStatisticInfo(SSmlHandle *info) { @@ -1521,6 +1532,8 @@ static void smlPrintStatisticInfo(SSmlHandle *info) { } int32_t smlClearForRerun(SSmlHandle *info) { + int32_t code = 0; + int32_t lino = 0; info->reRun = false; taosHashClear(info->childTables); @@ -1528,33 +1541,23 @@ int32_t smlClearForRerun(SSmlHandle *info) { taosHashClear(info->tableUids); if (!info->dataFormat) { - if (unlikely(info->lines != NULL)) { - uError("SML:0x%" PRIx64 " info->lines != NULL", info->id); - return TSDB_CODE_SML_INVALID_DATA; - } info->lines = (SSmlLineInfo *)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo)); - if (unlikely(info->lines == NULL)) { - uError("SML:0x%" PRIx64 " info->lines == NULL", info->id); - return terrno; - } + SML_CHECK_NULL(info->lines); } taosArrayClearP(info->escapedStringList, taosMemoryFree); + taosMemoryFreeClear(info->preLine.tags); (void)memset(&info->preLine, 0, sizeof(SSmlLineInfo)); info->currSTableMeta = NULL; info->currTableDataCtx = NULL; SVnodeModifyOpStmt *stmt = (SVnodeModifyOpStmt *)(info->pQuery->pRoot); - if (stmt == NULL){ - return TSDB_CODE_SML_INVALID_DATA; - } stmt->freeHashFunc(stmt->pTableBlockHashObj); stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (stmt->pTableBlockHashObj == NULL) { - uError("SML:0x%" PRIx64 " stmt->pTableBlockHashObj == NULL", info->id); - return terrno; - } - return TSDB_CODE_SUCCESS; + SML_CHECK_NULL(stmt->pTableBlockHashObj); + +END: + RETURN } static void printRaw(int64_t id, int lineNum, int numLines, ELogLevel level, char* data, int32_t len){ @@ -1667,6 +1670,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t retryNum = 0; info->cost.parseTime = taosGetTimestampUs(); @@ -1694,7 +1698,7 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL SML_CHECK_CODE(smlInsertData(info)); END: - return code; + RETURN } void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawLineEnd) { @@ -1738,6 +1742,7 @@ void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawL TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, char *rawLineEnd, int numLines, int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SRequestObj *request = NULL; SSmlHandle *info = NULL; int cnt = 0; diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index d44f9cac9c..feb0b4645a 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -290,7 +290,12 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ } static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { + if (is_same_child_table_telnet(elements, &info->preLine) == 0) { + elements->measureTag = info->preLine.measureTag; + return TSDB_CODE_SUCCESS; + } int32_t code = 0; + int32_t lino = 0; if(info->dataFormat){ SML_CHECK_CODE(smlProcessSuperTable(info, elements)); } @@ -302,7 +307,7 @@ END: if(info->reRun){ return TSDB_CODE_SUCCESS; } - return code; + RETURN } static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) { @@ -420,7 +425,8 @@ static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *timestamp) { } static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo *elements) { - int32_t ret = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; cJSON *metricJson = NULL; cJSON *tsJson = NULL; @@ -435,52 +441,22 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo } cJSON **marks[OTD_JSON_FIELDS_NUM] = {&metricJson, &tsJson, &valueJson, &tagsJson}; - ret = smlGetJsonElements(root, marks); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - return ret; - } - + SML_CHECK_CODE(smlGetJsonElements(root, marks)); // Parse metric - ret = smlParseMetricFromJSON(info, metricJson, elements); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - uError("OTD:0x%" PRIx64 " Unable to parse metric from JSON payload", info->id); - return ret; - } - + SML_CHECK_CODE(smlParseMetricFromJSON(info, metricJson, elements)); // Parse metric value SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN}; - ret = smlParseValueFromJSON(valueJson, &kv); - if (unlikely(ret)) { - uError("OTD:0x%" PRIx64 " Unable to parse metric value from JSON payload", info->id); - return ret; - } + SML_CHECK_CODE(smlParseValueFromJSON(valueJson, &kv)); // Parse tags - bool needFree = info->dataFormat; elements->tags = cJSON_PrintUnformatted(tagsJson); - if (elements->tags == NULL){ - return TSDB_CODE_OUT_OF_MEMORY; - } - elements->tagsLen = strlen(elements->tags); - if (is_same_child_table_telnet(elements, &info->preLine) != 0) { - ret = smlParseTagsFromJSON(info, tagsJson, elements); - if (unlikely(ret)) { - uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); - taosMemoryFree(elements->tags); - elements->tags = NULL; - return ret; - } - } else { - elements->measureTag = info->preLine.measureTag; - } + SML_CHECK_NULL(elements->tags); - if (needFree) { - taosMemoryFree(elements->tags); - elements->tags = NULL; - } + elements->tagsLen = strlen(elements->tags); + SML_CHECK_CODE(smlParseTagsFromJSON(info, tagsJson, elements)); if (unlikely(info->reRun)) { - return TSDB_CODE_SUCCESS; + goto END; } // Parse timestamp @@ -489,22 +465,29 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo if (unlikely(ts < 0)) { char* tmp = cJSON_PrintUnformatted(tsJson); if (tmp == NULL) { - uError("cJSON_PrintUnformatted failed since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %" PRId64, info->id, info->msgBuf.buf, ts); } else { uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts); taosMemoryFree(tmp); } - return TSDB_CODE_INVALID_TIMESTAMP; + code = TSDB_CODE_INVALID_TIMESTAMP; + goto END; } SSmlKv kvTs = {0}; smlBuildTsKv(&kvTs, ts); if (info->dataFormat){ - ret = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv); + code = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv); } else { - ret = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv); + code = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv); } - return ret; + SML_CHECK_CODE(code); + taosMemoryFreeClear(info->preLine.tags); + info->preLine = *elements; + elements->tags = NULL; + +END: + taosMemoryFree(elements->tags); + RETURN } int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { @@ -527,23 +510,7 @@ int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { return TSDB_CODE_TSC_INVALID_JSON; } - if (unlikely(info->lines != NULL)) { - for (int i = 0; i < info->lineNum; i++) { - taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv); - if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag); - } - taosMemoryFree(info->lines); - info->lines = NULL; - } info->lineNum = payloadNum; - info->dataFormat = true; - - ret = smlClearForRerun(info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - info->parseJsonByLib = true; cJSON *head = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : info->root->child; int cnt = 0; @@ -552,6 +519,8 @@ int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { if (info->dataFormat) { SSmlLineInfo element = {0}; ret = smlParseJSONStringExt(info, dataPoint, &element); + if (element.measureTagsLen != 0) taosMemoryFree(element.measureTag); + } else { ret = smlParseJSONStringExt(info, dataPoint, info->lines + cnt); } diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 5ecf4e2206..e2e60ee7c4 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -312,6 +312,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) { int32_t code = 0; + int32_t lino = 0; bool isSameCTable = IS_SAME_CHILD_TABLE; if(isSameCTable){ return TSDB_CODE_SUCCESS; @@ -327,7 +328,7 @@ END: if(info->reRun){ return TSDB_CODE_SUCCESS; } - return code; + RETURN } static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) { diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index bf422675ef..4deb6a34cd 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -149,19 +149,20 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS } int32_t code = 0; + int32_t lino = 0; if(info->dataFormat){ SML_CHECK_CODE(smlProcessSuperTable(info, elements)); } SML_CHECK_CODE(smlProcessTagTelnet(info, data, sqlEnd)); SML_CHECK_CODE(smlJoinMeasureTag(elements)); - return smlProcessChildTable(info, elements); + code = smlProcessChildTable(info, elements); END: if(info->reRun){ return TSDB_CODE_SUCCESS; } - return code; + RETURN } // format: =[ =] @@ -233,5 +234,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine } else { ret = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv); } + info->preLine = *elements; + return ret; } \ No newline at end of file diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 1d8d82ccb9..20f2d7dc25 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -105,6 +105,102 @@ int smlProcess_telnet_Test() { return code; } +int smlProcess_json0_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}}]"}; + + char *sql1[1] = {0}; + for (int i = 0; i < 1; i++) { + sql1[i] = taosMemoryCalloc(1, 1024); + ASSERT(sql1[i] != NULL); + (void)strncpy(sql1[i], sql[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + int code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql1[i]); + } + ASSERT(code == 0); + + + const char *sql2[] = { + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":1}" + "},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}" + "}]", + }; + + char *sql3[1] = {0}; + for (int i = 0; i < 1; i++) { + sql3[i] = taosMemoryCalloc(1, 1024); + ASSERT(sql3[i] != NULL); + (void)strncpy(sql3[i], sql2[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql3[i]); + } + + ASSERT(code == 0); + + + // TD-22903 + const char *sql4[] = { + "[{\"metric\": \"test_us\", \"timestamp\": {\"value\": 1626006833639, \"type\": \"ms\"}, \"value\": true, \"tags\": {\"t0\": true}}, {\"metric\": \"test_us\", \"timestamp\": {\"value\": 1626006833638, \"type\": \"ms\"}, \"value\": false, \"tags\": {\"t0\": true}}]" + }; + char *sql5[1] = {0}; + for (int i = 0; i < 1; i++) { + sql5[i] = taosMemoryCalloc(1, 1024); + ASSERT(sql5[i] != NULL); + (void)strncpy(sql5[i], sql4[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql5[i]); + } + ASSERT(code == 0); + + taos_close(taos); + + return code; +} + int smlProcess_json1_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -2135,8 +2231,8 @@ int main(int argc, char *argv[]) { taos_options(TSDB_OPTION_CONFIGDIR, argv[1]); } - int ret = 0; - ret = sml_ts5528_test(); +// int ret = smlProcess_json0_Test(); + int ret = sml_ts5528_test(); ASSERT(!ret); ret = sml_td29691_Test(); ASSERT(ret); @@ -2146,8 +2242,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_td18789_Test(); ASSERT(!ret); - ret = sml_td24070_Test(); - ASSERT(!ret); +// ret = sml_td24070_Test(); +// ASSERT(!ret); ret = sml_td23881_Test(); ASSERT(ret); ret = sml_escape_Test(); From 4090aad6fd2c53a940e9472a8d3614357d557704 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 24 Oct 2024 16:18:39 +0800 Subject: [PATCH 10/34] enh:[TD-32166]refactor code in sml --- source/client/src/clientSml.c | 8 +++- utils/test/c/sml_test.c | 82 ++++++++++++++++++++--------------- 2 files changed, 54 insertions(+), 36 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 5f4327b1cd..1b4643a958 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1289,7 +1289,9 @@ void smlDestroyInfo(SSmlHandle *info) { } taosMemoryFree(info->lines); } - taosMemoryFreeClear(info->preLine.tags); + if(info->protocol == TSDB_SML_JSON_PROTOCOL) { + taosMemoryFreeClear(info->preLine.tags); + } cJSON_Delete(info->root); taosMemoryFreeClear(info); } @@ -1546,7 +1548,9 @@ int32_t smlClearForRerun(SSmlHandle *info) { } taosArrayClearP(info->escapedStringList, taosMemoryFree); - taosMemoryFreeClear(info->preLine.tags); + if(info->protocol == TSDB_SML_JSON_PROTOCOL) { + taosMemoryFreeClear(info->preLine.tags); + } (void)memset(&info->preLine, 0, sizeof(SSmlLineInfo)); info->currSTableMeta = NULL; info->currTableDataCtx = NULL; diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 20f2d7dc25..6d4192280c 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -105,6 +105,44 @@ int smlProcess_telnet_Test() { return code; } +int smlProcess_telnet0_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql1[] = {"sysif.bytes.out 1479496100 1.3E0 host=web01 interface=eth0"}; + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_TELNET_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + ASSERT(code == 0); + taos_free_result(pRes); + + const char *sql2[] = {"sysif.bytes.out 1479496700 1.6E0 host=web01 interface=eth0"}; + pRes = taos_schemaless_insert(taos, (char **)sql2, sizeof(sql2) / sizeof(sql2[0]), TSDB_SML_TELNET_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + ASSERT(code == 0); + taos_free_result(pRes); + + const char *sql3[] = {"sysif.bytes.out 1479496300 1.1E0 interface=eth0 host=web01"}; + pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_TELNET_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + ASSERT(code == 0); + taos_free_result(pRes); + + taos_close(taos); + + return code; +} + int smlProcess_json0_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -115,7 +153,7 @@ int smlProcess_json0_Test() { taos_free_result(pRes); const char *sql[] = { - "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}}]"}; + "[{\"metric\":\"syscpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}}]"}; char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { @@ -141,8 +179,8 @@ int smlProcess_json0_Test() { const char *sql2[] = { - "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":1}" - "},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}" + "[{\"metric\":\"syscpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":1}" + "},{\"metric\":\"syscpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}" "}]", }; @@ -169,33 +207,6 @@ int smlProcess_json0_Test() { ASSERT(code == 0); - - // TD-22903 - const char *sql4[] = { - "[{\"metric\": \"test_us\", \"timestamp\": {\"value\": 1626006833639, \"type\": \"ms\"}, \"value\": true, \"tags\": {\"t0\": true}}, {\"metric\": \"test_us\", \"timestamp\": {\"value\": 1626006833638, \"type\": \"ms\"}, \"value\": false, \"tags\": {\"t0\": true}}]" - }; - char *sql5[1] = {0}; - for (int i = 0; i < 1; i++) { - sql5[i] = taosMemoryCalloc(1, 1024); - ASSERT(sql5[i] != NULL); - (void)strncpy(sql5[i], sql4[i], 1023); - } - - pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL, - TSDB_SML_TIMESTAMP_NANO_SECONDS); - code = taos_errno(pRes); - if (code != 0) { - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - } else { - printf("%s result:success\n", __FUNCTION__); - } - taos_free_result(pRes); - - for (int i = 0; i < 1; i++) { - taosMemoryFree(sql5[i]); - } - ASSERT(code == 0); - taos_close(taos); return code; @@ -2231,8 +2242,9 @@ int main(int argc, char *argv[]) { taos_options(TSDB_OPTION_CONFIGDIR, argv[1]); } -// int ret = smlProcess_json0_Test(); - int ret = sml_ts5528_test(); + int ret = smlProcess_json0_Test(); + ASSERT(!ret); + ret = sml_ts5528_test(); ASSERT(!ret); ret = sml_td29691_Test(); ASSERT(ret); @@ -2252,8 +2264,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_ts3116_Test(); ASSERT(!ret); - ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file - ASSERT(!ret); +// ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file +// ASSERT(!ret); ret = sml_ts3303_Test(); ASSERT(!ret); ret = sml_ttl_Test(); @@ -2268,6 +2280,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = smlProcess_telnet_Test(); ASSERT(!ret); + ret = smlProcess_telnet0_Test(); + ASSERT(!ret); ret = smlProcess_json1_Test(); ASSERT(!ret); ret = smlProcess_json2_Test(); From 35d3d968ff7e582d11808cdc74c96539516ee0d6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 24 Oct 2024 18:55:37 +0800 Subject: [PATCH 11/34] enh:[TD-32166]refactor code in sml --- source/client/inc/clientSml.h | 1 - source/client/src/clientSml.c | 101 +++++++++++++------------- source/client/src/clientSmlJson.c | 33 ++++----- source/client/src/clientSmlLine.c | 29 +++----- source/client/src/clientSmlTelnet.c | 16 ++-- source/client/test/smlTest.cpp | 9 +++ source/libs/parser/src/parInsertSml.c | 6 +- utils/test/c/sml_test.c | 24 ++++-- 8 files changed, 120 insertions(+), 99 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 8558ec46dc..d94a671c45 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -211,7 +211,6 @@ typedef struct { cJSON *root; // for parse json int8_t offset[OTD_JSON_FIELDS_NUM]; SSmlLineInfo *lines; // element is SSmlLineInfo - bool parseJsonByLib; SArray *tagJsonArray; SArray *valueJsonArray; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 1b4643a958..f9ba8531f4 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -169,23 +169,23 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u } int32_t smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen, SSmlTableInfo **tInfo) { + int32_t code = 0; + int32_t lino = 0; SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); - if (!tag) { - return terrno; - } + SML_CHECK_NULL(tag) tag->sTableName = measure; tag->sTableNameLen = measureLen; tag->cols = taosArrayInit(numRows, POINTER_BYTES); - if (tag->cols == NULL) { - uError("SML:smlBuildTableInfo failed to allocate memory"); - taosMemoryFree(tag); - return terrno; - } - + SML_CHECK_NULL(tag->cols) *tInfo = tag; - return TSDB_CODE_SUCCESS; + return code; + +END: + taosMemoryFree(tag); + uError("%s failed code:%d line:%d", __FUNCTION__ , code, lino); + return code; } void smlBuildTsKv(SSmlKv *kv, int64_t ts) { @@ -437,7 +437,7 @@ END: int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) { if (info->dataFormat) { - uDebug("SML:0x%" PRIx64 " smlParseEndLine format true, ts:%" PRId64, info->id, kvTs->i); + uDebug("SML:0x%" PRIx64 " %s format true, ts:%" PRId64, info->id, __FUNCTION__, kvTs->i); int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0); if (ret == TSDB_CODE_SUCCESS) { ret = smlBuildRow(info->currTableDataCtx); @@ -446,11 +446,11 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) clearColValArraySml(info->currTableDataCtx->pValues); taosArrayClearP(info->escapedStringList, taosMemoryFree); if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + uError("SML:0x%" PRIx64 " %s smlBuildCol error:%d", info->id, __FUNCTION__, ret); return ret; } } else { - uDebug("SML:0x%" PRIx64 " smlParseEndLine format false, ts:%" PRId64, info->id, kvTs->i); + uDebug("SML:0x%" PRIx64 " %s format false, ts:%" PRId64, info->id, __FUNCTION__, kvTs->i); taosArraySet(elements->colArray, 0, kvTs); } info->preLine = *elements; @@ -1134,7 +1134,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { taosHashCleanup(hashTmp); hashTmp = NULL; } else { - uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code)); + uError("SML:0x%" PRIx64 " %s load table meta error: %s", info->id, __FUNCTION__, tstrerror(code)); goto END; } @@ -1145,11 +1145,11 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { taosMemoryFreeClear(sTableData->tableMeta); sTableData->tableMeta = pTableMeta; - uDebug("SML:0x%" PRIx64 "modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, pTableMeta->uid, + uDebug("SML:0x%" PRIx64 " %s modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, __FUNCTION__, pTableMeta->uid, pTableMeta->sversion, pTableMeta->tversion); tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp); } - uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas end success, format:%d, needModifySchema:%d", info->id, info->dataFormat, + uDebug("SML:0x%" PRIx64 " %s end success, format:%d, needModifySchema:%d", info->id, __FUNCTION__, info->dataFormat, info->needModifySchema); return TSDB_CODE_SUCCESS; @@ -1159,7 +1159,7 @@ END: taosHashCleanup(hashTmp); taosMemoryFreeClear(pTableMeta); (void)catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); // ignore refresh meta code if there is an error - uError("SML:0x%" PRIx64 " smlModifyDBSchemas end failed:%d:%s, format:%d, needModifySchema:%d", info->id, code, + uError("SML:0x%" PRIx64 " %s end failed:%d:%s, format:%d, needModifySchema:%d", info->id, __FUNCTION__, code, tstrerror(code), info->dataFormat, info->needModifySchema); return code; @@ -1331,7 +1331,7 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { SML_CHECK_NULL(info->escapedStringList); *handle = info; - return code; + info = NULL; END: smlDestroyInfo(info); @@ -1362,8 +1362,8 @@ END: RETURN } -static int32_t smlParseLineBottom(SSmlHandle *info) { - uDebug("SML:0x%" PRIx64 " smlParseLineBottom start, format:%d, linenum:%d", info->id, info->dataFormat, +static int32_t smlParseEnd(SSmlHandle *info) { + uDebug("SML:0x%" PRIx64 " %s start, format:%d, linenum:%d", info->id, __FUNCTION__, info->dataFormat, info->lineNum); int32_t code = 0; int32_t lino = 0; @@ -1403,28 +1403,28 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { SSmlSTableMeta **tableMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); if (tableMeta) { // update meta - uDebug("SML:0x%" PRIx64 " smlParseLineBottom update meta, format:%d, linenum:%d", info->id, info->dataFormat, + uDebug("SML:0x%" PRIx64 " %s update meta, format:%d, linenum:%d", info->id, __FUNCTION__, info->dataFormat, info->lineNum); SML_CHECK_CODE(smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf, (*tableMeta)->tagHash)); SML_CHECK_CODE(smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf, (*tableMeta)->colHash)); } else { - uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, + uDebug("SML:0x%" PRIx64 " %s add meta, format:%d, linenum:%d", info->id, __FUNCTION__, info->dataFormat, info->lineNum); SSmlSTableMeta *meta = NULL; SML_CHECK_CODE(smlBuildSTableMeta(info->dataFormat, &meta)); code = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); if (code != TSDB_CODE_SUCCESS) { smlDestroySTableMeta(&meta); - uError("SML:0x%" PRIx64 " put measuer to hash failed", info->id); + uError("SML:0x%" PRIx64 " put measure to hash failed", info->id); goto END; } SML_CHECK_CODE(smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags, NULL)); SML_CHECK_CODE(smlInsertMeta(meta->colHash, meta->cols, elements->colArray, meta->tagHash)); } } - uDebug("SML:0x%" PRIx64 " smlParseLineBottom end, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); + uDebug("SML:0x%" PRIx64 " %s end, format:%d, linenum:%d", info->id, __FUNCTION__, info->dataFormat, info->lineNum); END: RETURN @@ -1435,7 +1435,7 @@ static int32_t smlInsertData(SSmlHandle *info) { int32_t lino = 0; char *measure = NULL; SSmlTableInfo **oneTable = NULL; - uDebug("SML:0x%" PRIx64 " smlInsertData start, format:%d", info->id, info->dataFormat); + uDebug("SML:0x%" PRIx64 " %s start, format:%d", info->id, __FUNCTION__, info->dataFormat); if (info->pRequest->dbList == NULL) { info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN); @@ -1484,7 +1484,7 @@ static int32_t smlInsertData(SSmlHandle *info) { SSmlSTableMeta **pMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); if (unlikely(NULL == pMeta || NULL == *pMeta || NULL == (*pMeta)->tableMeta)) { - uError("SML:0x%" PRIx64 " NULL == pMeta. table name: %s", info->id, tableData->childTableName); + uError("SML:0x%" PRIx64 " %s NULL == pMeta. table name: %s", info->id, __FUNCTION__, tableData->childTableName); code = TSDB_CODE_SML_INTERNAL_ERROR; goto END; } @@ -1492,7 +1492,7 @@ static int32_t smlInsertData(SSmlHandle *info) { // use tablemeta of stable to save vgid and uid of child table (*pMeta)->tableMeta->vgId = vg.vgId; (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid - uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname, + uDebug("SML:0x%" PRIx64 " %s table:%s, uid:%" PRIu64 ", format:%d", info->id, __FUNCTION__, pName.tname, tableData->uid, info->dataFormat); SML_CHECK_CODE(smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, @@ -1510,7 +1510,7 @@ static int32_t smlInsertData(SSmlHandle *info) { launchQueryImpl(info->pRequest, info->pQuery, true, NULL); // no need to check return code - uDebug("SML:0x%" PRIx64 " smlInsertData end, format:%d, code:%d,%s", info->id, info->dataFormat, info->pRequest->code, + uDebug("SML:0x%" PRIx64 " %s end, format:%d, code:%d,%s", info->id, __FUNCTION__, info->dataFormat, info->pRequest->code, tstrerror(info->pRequest->code)); return info->pRequest->code; @@ -1605,20 +1605,25 @@ static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLi return true; } -static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { - uDebug("SML:0x%" PRIx64 " smlParseLine start", info->id); + +static int32_t smlParseJson(SSmlHandle *info, char *lines[], char *rawLine) { + int32_t code = TSDB_CODE_SUCCESS; + if (lines) { + code = smlParseJSONExt(info, *lines); + } else if (rawLine) { + code = smlParseJSONExt(info, rawLine); + } + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed code:%d", __FUNCTION__ , code); + } + return code; +} + +static int32_t smlParseStart(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { + uDebug("SML:0x%" PRIx64 " %s start", info->id, __FUNCTION__); int32_t code = TSDB_CODE_SUCCESS; if (info->protocol == TSDB_SML_JSON_PROTOCOL) { - if (lines) { - code = smlParseJSONExt(info, *lines); - } else if (rawLine) { - code = smlParseJSONExt(info, rawLine); - } - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseJSON failed:%s", info->id, lines ? *lines : rawLine); - return code; - } - return code; + return smlParseJson(info, lines, rawLine); } char *oldRaw = rawLine; @@ -1644,19 +1649,17 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } else { code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, info->lines + i); } - } else { - code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; } if (code != TSDB_CODE_SUCCESS) { if (rawLine != NULL) { printRaw(info->id, i, numLines, DEBUG_ERROR, tmp, len); } else { - uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); + uError("SML:0x%" PRIx64 " %s failed. line %d : %s", info->id, __FUNCTION__, i, tmp); } return code; } if (info->reRun) { - uDebug("SML:0x%" PRIx64 " smlParseLine re run", info->id); + uDebug("SML:0x%" PRIx64 " %s re run", info->id, __FUNCTION__); i = 0; rawLine = oldRaw; code = smlClearForRerun(info); @@ -1667,7 +1670,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } i++; } - uDebug("SML:0x%" PRIx64 " smlParseLine end", info->id); + uDebug("SML:0x%" PRIx64 " %s end", info->id, __FUNCTION__); return code; } @@ -1679,8 +1682,8 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL info->cost.parseTime = taosGetTimestampUs(); - SML_CHECK_CODE(smlParseLine(info, lines, rawLine, rawLineEnd, numLines)); - SML_CHECK_CODE(smlParseLineBottom(info)); + SML_CHECK_CODE(smlParseStart(info, lines, rawLine, rawLineEnd, numLines)); + SML_CHECK_CODE(smlParseEnd(info)); info->cost.lineNum = info->lineNum; info->cost.numOfSTables = taosHashGetSize(info->superTables); @@ -1752,7 +1755,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, int cnt = 0; while (1) { SML_CHECK_CODE(createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid, &request)); - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; + SSmlMsgBuf msg = {request->msgBufLen, request->msgBuf}; request->code = smlBuildSmlInfo(taos, &info); SML_CHECK_CODE(request->code); @@ -1822,7 +1825,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, break; } - END: +END: smlDestroyInfo(info); return (TAOS_RES *)request; } diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index feb0b4645a..2dcc36eada 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -24,7 +24,7 @@ static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { elements->measureLen = strlen(metric->valuestring); if (IS_INVALID_TABLE_LEN(elements->measureLen)) { - uError("OTD:0x%" PRIx64 " Metric length is 0 or large than 192", info->id); + uError("SML:0x%" PRIx64 " Metric length is 0 or large than 192", info->id); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } @@ -44,7 +44,7 @@ static int32_t smlGetJsonElements(cJSON *root, cJSON ***marks) { child = child->next; } if (*marks[i] == NULL) { - uError("smlGetJsonElements error, not find mark:%d:%s", i, jsonName[i]); + uError("SML %s error, not find mark:%d:%s", __FUNCTION__, i, jsonName[i]); return TSDB_CODE_TSC_INVALID_JSON; } } @@ -53,7 +53,7 @@ static int32_t smlGetJsonElements(cJSON *root, cJSON ***marks) { static int32_t smlConvertJSONBool(SSmlKv *pVal, char *typeStr, cJSON *value) { if (strcasecmp(typeStr, "bool") != 0) { - uError("OTD:invalid type(%s) for JSON Bool", typeStr); + uError("SML:invalid type(%s) for JSON Bool", typeStr); return TSDB_CODE_TSC_INVALID_JSON_TYPE; } pVal->type = TSDB_DATA_TYPE_BOOL; @@ -67,7 +67,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { // tinyint if (strcasecmp(typeStr, "i8") == 0 || strcasecmp(typeStr, "tinyint") == 0) { if (!IS_VALID_TINYINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble); + uError("SML:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } pVal->type = TSDB_DATA_TYPE_TINYINT; @@ -78,7 +78,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { // smallint if (strcasecmp(typeStr, "i16") == 0 || strcasecmp(typeStr, "smallint") == 0) { if (!IS_VALID_SMALLINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(smallint)", value->valuedouble); + uError("SML:JSON value(%f) cannot fit in type(smallint)", value->valuedouble); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } pVal->type = TSDB_DATA_TYPE_SMALLINT; @@ -89,7 +89,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { // int if (strcasecmp(typeStr, "i32") == 0 || strcasecmp(typeStr, "int") == 0) { if (!IS_VALID_INT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(int)", value->valuedouble); + uError("SML:JSON value(%f) cannot fit in type(int)", value->valuedouble); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } pVal->type = TSDB_DATA_TYPE_INT; @@ -113,7 +113,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { // float if (strcasecmp(typeStr, "f32") == 0 || strcasecmp(typeStr, "float") == 0) { if (!IS_VALID_FLOAT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(float)", value->valuedouble); + uError("SML:JSON value(%f) cannot fit in type(float)", value->valuedouble); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } pVal->type = TSDB_DATA_TYPE_FLOAT; @@ -130,7 +130,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { } // if reach here means type is unsupported - uError("OTD:invalid type(%s) for JSON Number", typeStr); + uError("SML:invalid type(%s) for JSON Number", typeStr); return TSDB_CODE_TSC_INVALID_JSON_TYPE; } @@ -142,7 +142,7 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { } else if (strcasecmp(typeStr, "nchar") == 0) { pVal->type = TSDB_DATA_TYPE_NCHAR; } else { - uError("OTD:invalid type(%s) for JSON String", typeStr); + uError("SML:invalid type(%s) for JSON String", typeStr); return TSDB_CODE_TSC_INVALID_JSON_TYPE; } pVal->length = strlen(value->valuestring); @@ -225,7 +225,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { case cJSON_String: { int32_t ret = smlConvertJSONString(kv, "binary", root); if (ret != TSDB_CODE_SUCCESS) { - uError("OTD:Failed to parse binary value from JSON Obj"); + uError("SML:Failed to parse binary value from JSON Obj"); return ret; } break; @@ -233,7 +233,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { case cJSON_Object: { int32_t ret = smlParseValueFromJSONObj(root, kv); if (ret != TSDB_CODE_SUCCESS) { - uError("OTD:Failed to parse value from JSON Obj"); + uError("SML:Failed to parse value from JSON Obj"); return ret; } break; @@ -262,7 +262,7 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ } size_t keyLen = strlen(tag->string); if (unlikely(IS_INVALID_COL_LEN(keyLen))) { - uError("OTD:Tag key length is 0 or too large than 64"); + uError("SML:Tag key length is 0 or too large than 64"); return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } @@ -436,7 +436,7 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo int32_t size = cJSON_GetArraySize(root); // outmost json fields has to be exactly 4 if (size != OTD_JSON_FIELDS_NUM) { - uError("OTD:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size); + uError("SML:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size); return TSDB_CODE_TSC_INVALID_JSON; } @@ -465,9 +465,9 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo if (unlikely(ts < 0)) { char* tmp = cJSON_PrintUnformatted(tsJson); if (tmp == NULL) { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %" PRId64, info->id, info->msgBuf.buf, ts); + uError("SML:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %" PRId64, info->id, info->msgBuf.buf, ts); } else { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts); + uError("SML:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts); taosMemoryFree(tmp); } code = TSDB_CODE_INVALID_TIMESTAMP; @@ -506,7 +506,7 @@ int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { } else if (cJSON_IsObject(info->root)) { payloadNum = 1; } else { - uError("SML:0x%" PRIx64 " Invalid JSON Payload 3:%s", info->id, payload); + uError("SML:0x%" PRIx64 " Invalid JSON type:%s", info->id, payload); return TSDB_CODE_TSC_INVALID_JSON; } @@ -520,7 +520,6 @@ int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { SSmlLineInfo element = {0}; ret = smlParseJSONStringExt(info, dataPoint, &element); if (element.measureTagsLen != 0) taosMemoryFree(element.measureTag); - } else { ret = smlParseJSONStringExt(info, dataPoint, info->lines + cnt); } diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index e2e60ee7c4..b54f4e0beb 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -63,7 +63,7 @@ static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t le int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision); if (unlikely(ts == -1)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line invalid timestamp", data); return TSDB_CODE_SML_INVALID_DATA; } return ts; @@ -84,7 +84,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { } if (pVal->value[0] == 'l' || pVal->value[0] == 'L') { // nchar - if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= 3) { + if (pVal->length >= NCHAR_ADD_LEN && pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"') { pVal->type = TSDB_DATA_TYPE_NCHAR; pVal->length -= NCHAR_ADD_LEN; if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { @@ -97,7 +97,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { } if (pVal->value[0] == 'g' || pVal->value[0] == 'G') { // geometry - if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= sizeof("POINT")+3) { + if (pVal->length >= NCHAR_ADD_LEN && pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"') { int32_t code = initCtxGeomFromText(); if (code != TSDB_CODE_SUCCESS) { return code; @@ -124,7 +124,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { } if (pVal->value[0] == 'b' || pVal->value[0] == 'B') { // varbinary - if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= 3) { + if (pVal->length >= NCHAR_ADD_LEN && pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"') { pVal->type = TSDB_DATA_TYPE_VARBINARY; if(isHex(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN)){ if(!isValidateHex(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN)){ @@ -346,7 +346,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL const char *escapeChar = NULL; while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line invalid data", *sql); return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(IS_EQUAL(*sql,escapeChar))) { @@ -363,7 +363,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL } if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line invalid key or key is too long than 64", key); return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } @@ -397,18 +397,18 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL valueLen = *sql - value; if (unlikely(quoteNum != 0 && quoteNum != 2)) { - smlBuildInvalidDataMsg(&info->msgBuf, "unbalanced quotes", value); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line unbalanced quotes", value); return TSDB_CODE_SML_INVALID_DATA; } if (unlikely(valueLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line invalid value", value); return TSDB_CODE_SML_INVALID_DATA; } SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; int32_t ret = smlParseValue(&kv, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); + uError("SML:0x%" PRIx64 " %s parse value error:%d.", info->id, __FUNCTION__, ret); return ret; } @@ -430,11 +430,6 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL } (void)memcpy(tmp, kv.value, kv.length); PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length); - if(kv.type == TSDB_DATA_TYPE_GEOMETRY) { - uError("SML:0x%" PRIx64 " smlParseColLine error, invalid GEOMETRY type.", info->id); - taosMemoryFree((void*)kv.value); - return TSDB_CODE_TSC_INVALID_VALUE; - } if(kv.type == TSDB_DATA_TYPE_VARBINARY){ taosMemoryFree((void*)kv.value); } @@ -503,7 +498,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine } elements->measureLen = sql - elements->measure; if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen - measureLenEscaped))) { - smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line measure is empty or too large than 192", NULL); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } @@ -550,7 +545,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine elements->colsLen = sql - elements->cols; if (unlikely(elements->colsLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "cols is empty", NULL); + smlBuildInvalidDataMsg(&info->msgBuf, "SML line cols is empty", NULL); return TSDB_CODE_SML_INVALID_DATA; } @@ -567,7 +562,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine int64_t ts = smlParseInfluxTime(info, elements->timestamp, elements->timestampLen); if (unlikely(ts <= 0)) { - uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); + uError("SML:0x%" PRIx64 " %s error:%" PRId64, info->id, __FUNCTION__, ts); return TSDB_CODE_INVALID_TIMESTAMP; } diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index 4deb6a34cd..dd264da11e 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -172,14 +172,14 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine // parse metric smlParseTelnetElement(&sql, sqlEnd, &elements->measure, &elements->measureLen); if (unlikely((!(elements->measure) || IS_INVALID_TABLE_LEN(elements->measureLen)))) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid measure", sql); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } // parse timestamp smlParseTelnetElement(&sql, sqlEnd, &elements->timestamp, &elements->timestampLen); if (unlikely(!elements->timestamp || elements->timestampLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid timestamp", sql); return TSDB_CODE_SML_INVALID_DATA; } @@ -189,19 +189,21 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine } int64_t ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen); if (unlikely(ts < 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet parse timestamp failed", sql); return TSDB_CODE_INVALID_TIMESTAMP; } // parse value smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen); if (unlikely(!elements->cols || elements->colsLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid value", sql); return TSDB_CODE_TSC_INVALID_VALUE; } SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; - if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { + int ret = smlParseValue(&kv, &info->msgBuf); + if (ret != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " %s parse value error:%d.", info->id, __FUNCTION__, ret); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -210,11 +212,11 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine elements->tags = sql; elements->tagsLen = sqlEnd - sql; if (unlikely(!elements->tags || elements->tagsLen == 0)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); + smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid tag value", sql); return TSDB_CODE_TSC_INVALID_VALUE; } - int ret = smlParseTelnetTags(info, sql, sqlEnd, elements); + ret = smlParseTelnetTags(info, sql, sqlEnd, elements); if (unlikely(ret != TSDB_CODE_SUCCESS)) { return ret; } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index bcd12a393a..338457bec4 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -68,6 +68,15 @@ TEST(testCase, smlParseInfluxString_Test) { taosArrayDestroy(elements.colArray); elements.colArray = nullptr; + // case 0 false + tmp = "st,t1=3 c3=\""; + (void)memcpy(sql, tmp, strlen(tmp) + 1); + (void)memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); + ASSERT_NE(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = nullptr; + // case 2 false tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; (void)memcpy(sql, tmp, strlen(tmp) + 1); diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index b1ff8cb733..b5cdf1e4ee 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -472,7 +472,7 @@ int32_t smlInitHandle(SQuery** query) { int32_t code = nodesMakeNode(QUERY_NODE_QUERY, (SNode**)&pQuery); if (code != 0) { - uError("create pQuery error"); + uError("SML create pQuery error"); goto END; } pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; @@ -480,12 +480,12 @@ int32_t smlInitHandle(SQuery** query) { pQuery->msgType = TDMT_VND_SUBMIT; code = nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT, (SNode**)&stmt); if (code != 0) { - uError("create SVnodeModifyOpStmt error"); + uError("SML create SVnodeModifyOpStmt error"); goto END; } stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (stmt->pTableBlockHashObj == NULL){ - uError("create pTableBlockHashObj error"); + uError("SML create pTableBlockHashObj error"); code = terrno; goto END; } diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 6d4192280c..a76dccbbc0 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1882,6 +1882,20 @@ int sml_td24559_Test() { pRes = taos_query(taos, "create database if not exists td24559"); taos_free_result(pRes); + const char *sql1[] = { + "sttb,t1=1 f1=283i32,f2=g\"\" 1632299372000", + "sttb,t1=1 f2=G\"Point(4.343 89.342)\",f1=106i32 1632299373000", + }; + + pRes = taos_query(taos, "use td24559"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + int code = taos_errno(pRes); + printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); + ASSERT(code); + const char *sql[] = { "stb,t1=1 f1=283i32,f2=g\"Point(4.343 89.342)\" 1632299372000", "stb,t1=1 f2=G\"Point(4.343 89.342)\",f1=106i32 1632299373000", @@ -1895,7 +1909,7 @@ int sml_td24559_Test() { pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); - int code = taos_errno(pRes); + code = taos_errno(pRes); printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); taos_free_result(pRes); @@ -2254,8 +2268,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_td18789_Test(); ASSERT(!ret); -// ret = sml_td24070_Test(); -// ASSERT(!ret); + ret = sml_td24070_Test(); + ASSERT(!ret); ret = sml_td23881_Test(); ASSERT(ret); ret = sml_escape_Test(); @@ -2264,8 +2278,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_ts3116_Test(); ASSERT(!ret); -// ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file -// ASSERT(!ret); + ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file + ASSERT(!ret); ret = sml_ts3303_Test(); ASSERT(!ret); ret = sml_ttl_Test(); From 6960a34cfbea562e98aaad277c0ffa003770423a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 24 Oct 2024 19:52:39 +0800 Subject: [PATCH 12/34] enh:[TD-32166]refactor code in sml --- source/client/src/clientSml.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index f9ba8531f4..82e58f1467 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -698,8 +698,8 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; if (index) { if (colField[*index].type != kv->type) { - snprintf(info->msgBuf.buf, info->msgBuf.len, "SML:0x%" PRIx64 " %s point type and db type mismatch. db type: %d, point type: %d, key: %s", - info->id, __FUNCTION__, colField[*index].type, kv->type, kv->key); + snprintf(info->msgBuf.buf, info->msgBuf.len, "SML:0x%" PRIx64 " %s point type and db type mismatch. db type: %s, point type: %s, key: %s", + info->id, __FUNCTION__, tDataTypes[colField[*index].type].name, tDataTypes[kv->type].name, kv->key); uError("%s", info->msgBuf.buf); return TSDB_CODE_SML_INVALID_DATA; } From bd81eeccd09b66ef0d60cae7a1b80b5be8c6e3f8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 24 Oct 2024 23:54:12 +0800 Subject: [PATCH 13/34] enh:[TD-32166]refactor code in sml --- utils/test/c/sml_test.c | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index a76dccbbc0..3387447059 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1895,6 +1895,7 @@ int sml_td24559_Test() { int code = taos_errno(pRes); printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); ASSERT(code); + taos_free_result(pRes); const char *sql[] = { "stb,t1=1 f1=283i32,f2=g\"Point(4.343 89.342)\" 1632299372000", From a062a0cb8453037b3924cf7a04760a44f742a93d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 25 Oct 2024 09:41:35 +0800 Subject: [PATCH 14/34] enh:[TD-32166]refactor code in sml --- source/client/src/clientSml.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 82e58f1467..9d3b28e105 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1851,7 +1851,7 @@ END: TAOS_RES *taos_schemaless_insert_ttl_with_reqid_tbname_key(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey) { - if (taos == NULL || lines == NULL || numLines <= 0) { + if (taos == NULL || lines == NULL || numLines < 0) { terrno = TSDB_CODE_INVALID_PARA; return NULL; } @@ -1901,7 +1901,7 @@ static int32_t getRawLineLen(char *lines, int len, int protocol) { TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid_tbname_key(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey) { - if (taos == NULL || lines == NULL || len <= 0) { + if (taos == NULL || lines == NULL || len < 0) { terrno = TSDB_CODE_INVALID_PARA; return NULL; } From 39af874ff8e8c50ae9efaeeee030518f5538914e Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 25 Oct 2024 14:20:07 +0800 Subject: [PATCH 15/34] enh(tsdb/cache): move last ref array into shard instance --- source/util/src/tlrucache.c | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index 69832cd46c..bccf37c7d6 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -14,12 +14,12 @@ */ #define _DEFAULT_SOURCE +#include "tlrucache.h" #include "os.h" #include "taoserror.h" #include "tarray.h" #include "tdef.h" #include "tlog.h" -#include "tlrucache.h" #include "tutil.h" typedef struct SLRUEntry SLRUEntry; @@ -248,6 +248,7 @@ struct SLRUCacheShard { SLRUEntryTable table; size_t usage; // Memory size for entries residing in the cache. size_t lruUsage; // Memory size for entries residing only in the LRU list. + SArray *lastReferenceList; TdThreadMutex mutex; }; @@ -367,25 +368,34 @@ static void taosLRUCacheShardCleanup(SLRUCacheShard *shard) { (void)taosThreadMutexDestroy(&shard->mutex); taosLRUEntryTableCleanup(&shard->table); + + taosArrayDestroy(shard->lastReferenceList); } static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry *e, LRUHandle **handle, bool freeOnFail) { LRUStatus status = TAOS_LRU_STATUS_OK; - SArray *lastReferenceList = taosArrayInit(16, POINTER_BYTES); - if (!lastReferenceList) { - taosLRUEntryFree(e); - return TAOS_LRU_STATUS_FAIL; - } (void)taosThreadMutexLock(&shard->mutex); - taosLRUCacheShardEvictLRU(shard, e->totalCharge, lastReferenceList); + if (!shard->lastReferenceList) { + shard->lastReferenceList = taosArrayInit(16, POINTER_BYTES); + if (!shard->lastReferenceList) { + taosLRUEntryFree(e); + (void)taosThreadMutexUnlock(&shard->mutex); + + return TAOS_LRU_STATUS_FAIL; + } + } else { + taosArrayClear(shard->lastReferenceList); + } + + taosLRUCacheShardEvictLRU(shard, e->totalCharge, shard->lastReferenceList); if (shard->usage + e->totalCharge > shard->capacity && (shard->strictCapacity || handle == NULL)) { TAOS_LRU_ENTRY_SET_IN_CACHE(e, false); if (handle == NULL) { - if (!taosArrayPush(lastReferenceList, &e)) { + if (!taosArrayPush(shard->lastReferenceList, &e)) { taosLRUEntryFree(e); goto _exit; } @@ -413,7 +423,7 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry * taosLRUCacheShardLRURemove(shard, old); shard->usage -= old->totalCharge; - if (!taosArrayPush(lastReferenceList, &old)) { + if (!taosArrayPush(shard->lastReferenceList, &old)) { taosLRUEntryFree(e); taosLRUEntryFree(old); goto _exit; @@ -434,12 +444,11 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry * _exit: (void)taosThreadMutexUnlock(&shard->mutex); - for (int i = 0; i < taosArrayGetSize(lastReferenceList); ++i) { - SLRUEntry *entry = taosArrayGetP(lastReferenceList, i); + for (int i = 0; i < taosArrayGetSize(shard->lastReferenceList); ++i) { + SLRUEntry *entry = taosArrayGetP(shard->lastReferenceList, i); taosLRUEntryFree(entry); } - taosArrayDestroy(lastReferenceList); return status; } @@ -733,7 +742,8 @@ void taosLRUCacheCleanup(SLRUCache *cache) { } LRUStatus taosLRUCacheInsert(SLRUCache *cache, const void *key, size_t keyLen, void *value, size_t charge, - _taos_lru_deleter_t deleter, _taos_lru_overwriter_t overwriter, LRUHandle **handle, LRUPriority priority, void *ud) { + _taos_lru_deleter_t deleter, _taos_lru_overwriter_t overwriter, LRUHandle **handle, + LRUPriority priority, void *ud) { uint32_t hash = TAOS_LRU_CACHE_SHARD_HASH32(key, keyLen); uint32_t shardIndex = hash & cache->shardedCache.shardMask; From 69e930be6646561d4e190961c9342602ae0a4cbc Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 28 Oct 2024 10:18:48 +0800 Subject: [PATCH 16/34] Revert "enh(tsdb/cache): move last ref array into shard instance" This reverts commit 39af874ff8e8c50ae9efaeeee030518f5538914e. --- source/util/src/tlrucache.c | 36 +++++++++++++----------------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index bccf37c7d6..69832cd46c 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -14,12 +14,12 @@ */ #define _DEFAULT_SOURCE -#include "tlrucache.h" #include "os.h" #include "taoserror.h" #include "tarray.h" #include "tdef.h" #include "tlog.h" +#include "tlrucache.h" #include "tutil.h" typedef struct SLRUEntry SLRUEntry; @@ -248,7 +248,6 @@ struct SLRUCacheShard { SLRUEntryTable table; size_t usage; // Memory size for entries residing in the cache. size_t lruUsage; // Memory size for entries residing only in the LRU list. - SArray *lastReferenceList; TdThreadMutex mutex; }; @@ -368,34 +367,25 @@ static void taosLRUCacheShardCleanup(SLRUCacheShard *shard) { (void)taosThreadMutexDestroy(&shard->mutex); taosLRUEntryTableCleanup(&shard->table); - - taosArrayDestroy(shard->lastReferenceList); } static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry *e, LRUHandle **handle, bool freeOnFail) { LRUStatus status = TAOS_LRU_STATUS_OK; + SArray *lastReferenceList = taosArrayInit(16, POINTER_BYTES); + if (!lastReferenceList) { + taosLRUEntryFree(e); + return TAOS_LRU_STATUS_FAIL; + } (void)taosThreadMutexLock(&shard->mutex); - if (!shard->lastReferenceList) { - shard->lastReferenceList = taosArrayInit(16, POINTER_BYTES); - if (!shard->lastReferenceList) { - taosLRUEntryFree(e); - (void)taosThreadMutexUnlock(&shard->mutex); - - return TAOS_LRU_STATUS_FAIL; - } - } else { - taosArrayClear(shard->lastReferenceList); - } - - taosLRUCacheShardEvictLRU(shard, e->totalCharge, shard->lastReferenceList); + taosLRUCacheShardEvictLRU(shard, e->totalCharge, lastReferenceList); if (shard->usage + e->totalCharge > shard->capacity && (shard->strictCapacity || handle == NULL)) { TAOS_LRU_ENTRY_SET_IN_CACHE(e, false); if (handle == NULL) { - if (!taosArrayPush(shard->lastReferenceList, &e)) { + if (!taosArrayPush(lastReferenceList, &e)) { taosLRUEntryFree(e); goto _exit; } @@ -423,7 +413,7 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry * taosLRUCacheShardLRURemove(shard, old); shard->usage -= old->totalCharge; - if (!taosArrayPush(shard->lastReferenceList, &old)) { + if (!taosArrayPush(lastReferenceList, &old)) { taosLRUEntryFree(e); taosLRUEntryFree(old); goto _exit; @@ -444,11 +434,12 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry * _exit: (void)taosThreadMutexUnlock(&shard->mutex); - for (int i = 0; i < taosArrayGetSize(shard->lastReferenceList); ++i) { - SLRUEntry *entry = taosArrayGetP(shard->lastReferenceList, i); + for (int i = 0; i < taosArrayGetSize(lastReferenceList); ++i) { + SLRUEntry *entry = taosArrayGetP(lastReferenceList, i); taosLRUEntryFree(entry); } + taosArrayDestroy(lastReferenceList); return status; } @@ -742,8 +733,7 @@ void taosLRUCacheCleanup(SLRUCache *cache) { } LRUStatus taosLRUCacheInsert(SLRUCache *cache, const void *key, size_t keyLen, void *value, size_t charge, - _taos_lru_deleter_t deleter, _taos_lru_overwriter_t overwriter, LRUHandle **handle, - LRUPriority priority, void *ud) { + _taos_lru_deleter_t deleter, _taos_lru_overwriter_t overwriter, LRUHandle **handle, LRUPriority priority, void *ud) { uint32_t hash = TAOS_LRU_CACHE_SHARD_HASH32(key, keyLen); uint32_t shardIndex = hash & cache->shardedCache.shardMask; From a5501f30461b19be30f15a47c3fa82f8f6f3cec9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 29 Oct 2024 10:07:08 +0800 Subject: [PATCH 17/34] test: add test cases. --- tests/script/tsim/stream/snodeCheck.sim | 64 +++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 tests/script/tsim/stream/snodeCheck.sim diff --git a/tests/script/tsim/stream/snodeCheck.sim b/tests/script/tsim/stream/snodeCheck.sim new file mode 100644 index 0000000000..f4ab8c8124 --- /dev/null +++ b/tests/script/tsim/stream/snodeCheck.sim @@ -0,0 +1,64 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 + +system sh/cfg.sh -n dnode1 -c supportVnodes -v 4 +system sh/cfg.sh -n dnode2 -c supportVnodes -v 4 +system sh/cfg.sh -n dnode3 -c supportVnodes -v 4 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ========== step2 +sql create dnode $hostname port 7200 +system sh/exec.sh -n dnode2 -s start + +sql create dnode $hostname port 7300 +system sh/exec.sh -n dnode3 -s start + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql select * from information_schema.ins_dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 3 then + return -1 +endi +if $data(1)[4] != ready then + goto step2 +endi +if $data(2)[4] != ready then + goto step2 +endi + +print ========== step3 +sql drop database if exists test; +sql create database if not exists test vgroups 4 replica 3 precision "ms" ; +sql use test; + +sql create table test.test (ts timestamp, c1 int) tags (t1 int) ; + +print create stream without snode existing +sql_error create stream stream_t1 trigger at_once into str_dst as select count(*) from test interval(20s); + +print create snode +sql create snode on dnode 1; + +sql create stream stream_t1 trigger at_once into str_dst as select count(*) from test interval(20s); + +print drop snode and then create stream +sql drop snode on dnode 1; + +sql_error create stream stream_t2 trigger at_once into str_dst as select count(*) from test interval(20s); + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT From 18c1b76cb4a20205f21dfe3af6464853729135b8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 29 Oct 2024 10:09:07 +0800 Subject: [PATCH 18/34] test: add into test suite --- tests/parallel_test/cases.task | 1 + tests/script/tsim/testsuit.sim | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 09216add82..2cf0ec7dc3 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1317,6 +1317,7 @@ ,,y,script,./test.sh -f tsim/stream/basic2.sim ,,y,script,./test.sh -f tsim/stream/basic3.sim ,,y,script,./test.sh -f tsim/stream/basic4.sim +,,y,script,./test.sh -f tsim/stream/snodeCheck.sim ,,y,script,./test.sh -f tsim/stream/checkpointInterval0.sim ,,y,script,./test.sh -f tsim/stream/checkStreamSTable1.sim ,,y,script,./test.sh -f tsim/stream/checkStreamSTable.sim diff --git a/tests/script/tsim/testsuit.sim b/tests/script/tsim/testsuit.sim index c208a07488..ff3bb82aaf 100644 --- a/tests/script/tsim/testsuit.sim +++ b/tests/script/tsim/testsuit.sim @@ -102,6 +102,7 @@ run tsim/stream/triggerInterval0.sim run tsim/stream/triggerSession0.sim run tsim/stream/distributeIntervalRetrive0.sim run tsim/stream/basic0.sim +run tsim/stream/snodeCheck.sim run tsim/stream/session0.sim run tsim/stream/schedSnode.sim run tsim/stream/partitionby.sim From cf3e84e79d88e4227aa40c6aabee1e3ab5199d36 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 29 Oct 2024 12:25:25 +0800 Subject: [PATCH 19/34] tsdb/cache: cache schema to reuse --- source/dnode/vnode/src/inc/tsdb.h | 4 +++- source/dnode/vnode/src/tsdb/tsdbCache.c | 26 +++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ed8f99ec75..f1ee5267a1 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -342,7 +342,9 @@ typedef struct { rocksdb_writeoptions_t *writeoptions; rocksdb_readoptions_t *readoptions; rocksdb_writebatch_t *writebatch; - TdThreadMutex writeBatchMutex; + TdThreadMutex writeBatchMutex; + tb_uid_t suid; + tb_uid_t uid; STSchema *pTSchema; } SRocksCache; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 5583e464ed..fa375a9fa0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -221,7 +221,7 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { rocksdb_writebatch_t *writebatch = rocksdb_writebatch_create(); - TAOS_CHECK_GOTO(taosThreadMutexInit(&pTsdb->rCache.writeBatchMutex, NULL), &lino, _err6) ; + TAOS_CHECK_GOTO(taosThreadMutexInit(&pTsdb->rCache.writeBatchMutex, NULL), &lino, _err6); pTsdb->rCache.writebatch = writebatch; pTsdb->rCache.my_comparator = cmp; @@ -230,6 +230,8 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { pTsdb->rCache.readoptions = readoptions; pTsdb->rCache.flushoptions = flushoptions; pTsdb->rCache.db = db; + pTsdb->rCache.suid = -1; + pTsdb->rCache.uid = -1; pTsdb->rCache.pTSchema = NULL; TAOS_RETURN(code); @@ -1302,6 +1304,22 @@ _exit: TAOS_RETURN(code); } +static int32_t tsdbUpdateSkm(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid) { + if (suid) { + if (pTsdb->rCache.suid == suid) { + pTsdb->rCache.uid = uid; + return 0; + } + } else if (pTsdb->rCache.uid == uid) { + return 0; + } + + pTsdb->rCache.suid = suid; + pTsdb->rCache.uid = uid; + tDestroyTSchema(pTsdb->rCache.pTSchema); + return metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, -1, &pTsdb->rCache.pTSchema); +} + int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int64_t version, int32_t nRow, SRow **aRow) { int32_t code = 0, lino = 0; @@ -1314,7 +1332,11 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 SArray *ctxArray = NULL; SSHashObj *iColHash = NULL; + TAOS_CHECK_GOTO(tsdbUpdateSkm(pTsdb, suid, uid), &lino, _exit); + pTSchema = pTsdb->rCache.pTSchema; + /* TAOS_CHECK_GOTO(metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema), &lino, _exit); + */ TSDBROW tRow = {.type = TSDBROW_ROW_FMT, .version = version}; int32_t nCol = pTSchema->numOfCols; @@ -1393,7 +1415,7 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 } _exit: - taosMemoryFreeClear(pTSchema); + // taosMemoryFreeClear(pTSchema); taosArrayDestroy(ctxArray); tSimpleHashCleanup(iColHash); From e364d5aa66479400f3e2fe2ccae136a1467042c1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 29 Oct 2024 14:41:14 +0800 Subject: [PATCH 20/34] tsdb/cache: invalidate cached schema --- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/meta/metaTable.c | 6 ++++++ source/dnode/vnode/src/tsdb/tsdbCache.c | 14 ++++++++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index fc98d6578b..7673e562a8 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -222,6 +222,7 @@ int32_t tsdbCacheNewSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, int8_t int32_t tsdbCacheDropSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, bool hasPrimayKey); int32_t tsdbCacheNewNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, int8_t col_type); int32_t tsdbCacheDropNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, bool hasPrimayKey); +void tsdbCacheInvalidateSchema(STsdb* pTsdb, tb_uid_t suid, tb_uid_t uid); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 21d12ef77d..f525a28bd6 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -620,6 +620,8 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { } } if (uids) taosArrayDestroy(uids); + + tsdbCacheInvalidateSchema(pTsdb, pReq->suid, -1); } metaWLock(pMeta); @@ -1945,6 +1947,10 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl break; } + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, entry.uid); + } + entry.version = version; // do actual write diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index fa375a9fa0..abc1e214a4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -493,7 +493,7 @@ int tsdbCacheFlushDirty(const void *key, size_t klen, void *value, void *ud) { int32_t tsdbCacheCommit(STsdb *pTsdb) { int32_t code = 0; char *err = NULL; - + /* SLRUCache *pCache = pTsdb->lruCache; rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; @@ -511,7 +511,7 @@ int32_t tsdbCacheCommit(STsdb *pTsdb) { rocksdb_free(err); code = TSDB_CODE_FAILED; } - + */ TAOS_RETURN(code); } @@ -1304,6 +1304,16 @@ _exit: TAOS_RETURN(code); } +void tsdbCacheInvalidateSchema(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid) { + if (suid) { + if (pTsdb->rCache.suid == suid) { + pTsdb->rCache.suid = -1; + } + } else if (pTsdb->rCache.uid == uid) { + pTsdb->rCache.uid = -1; + } +} + static int32_t tsdbUpdateSkm(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid) { if (suid) { if (pTsdb->rCache.suid == suid) { From 6b67c2ebf0be26e47810ce2719f3320b6a61f937 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 29 Oct 2024 15:07:13 +0800 Subject: [PATCH 21/34] uncomment tsdb cache commit --- source/dnode/vnode/src/tsdb/tsdbCache.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index abc1e214a4..801e89c2b6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -493,7 +493,7 @@ int tsdbCacheFlushDirty(const void *key, size_t klen, void *value, void *ud) { int32_t tsdbCacheCommit(STsdb *pTsdb) { int32_t code = 0; char *err = NULL; - /* + SLRUCache *pCache = pTsdb->lruCache; rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; @@ -511,7 +511,7 @@ int32_t tsdbCacheCommit(STsdb *pTsdb) { rocksdb_free(err); code = TSDB_CODE_FAILED; } - */ + TAOS_RETURN(code); } From 8b4b7c141e1d2adedf9b0b3e40194e8812cea7e6 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 30 Oct 2024 10:22:19 +0800 Subject: [PATCH 22/34] fix schema update with sver param --- source/dnode/vnode/src/tsdb/tsdbCache.c | 76 ++++++++++++------------- source/util/src/tlrucache.c | 20 +++++-- 2 files changed, 51 insertions(+), 45 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 801e89c2b6..16812b16f4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1136,19 +1136,17 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray (void)taosThreadMutexLock(&pTsdb->lruMutex); for (int i = 0; i < num_keys; ++i) { - SLastUpdateCtx *updCtx = (SLastUpdateCtx *)taosArrayGet(updCtxArray, i); - - int8_t lflag = updCtx->lflag; - SRowKey *pRowKey = &updCtx->tsdbRowKey.key; - SColVal *pColVal = &updCtx->colVal; + SLastUpdateCtx *updCtx = &((SLastUpdateCtx *)TARRAY_DATA(updCtxArray))[i]; + int8_t lflag = updCtx->lflag; + SRowKey *pRowKey = &updCtx->tsdbRowKey.key; + SColVal *pColVal = &updCtx->colVal; if (lflag == LFLAG_LAST && !COL_VAL_IS_VALUE(pColVal)) { continue; } SLastKey *key = &(SLastKey){.lflag = lflag, .uid = uid, .cid = pColVal->cid}; - size_t klen = ROCKS_KEY_LEN; - LRUHandle *h = taosLRUCacheLookup(pCache, key, klen); + LRUHandle *h = taosLRUCacheLookup(pCache, key, ROCKS_KEY_LEN); if (h) { SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h); if (pLastCol->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) { @@ -1304,28 +1302,30 @@ _exit: TAOS_RETURN(code); } -void tsdbCacheInvalidateSchema(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid) { - if (suid) { - if (pTsdb->rCache.suid == suid) { +void tsdbCacheInvalidateSchema(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int32_t sver) { + if (pTsdb->rCache.pTSchema && pTsdb->rCache.suid == suid) { + if (pTsdb->rCache.suid && sver == pTsdb->rCache.sver) { + pTsdb->rCache.sver = -1; pTsdb->rCache.suid = -1; + } else if (pTsdb->rCache.uid == uid && pTsdb->rCache.sver == sver) { + pTsdb->rCache.sver = -1; + pTsdb->rCache.uid = -1; } - } else if (pTsdb->rCache.uid == uid) { - pTsdb->rCache.uid = -1; } } -static int32_t tsdbUpdateSkm(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid) { - if (suid) { - if (pTsdb->rCache.suid == suid) { - pTsdb->rCache.uid = uid; +static int32_t tsdbUpdateSkm(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int32_t sver) { + if (pTsdb->rCache.pTSchema && pTsdb->rCache.suid == suid) { + if (pTsdb->rCache.suid && sver == pTsdb->rCache.sver) { + return 0; + } else if (pTsdb->rCache.uid == uid && pTsdb->rCache.sver == sver) { return 0; } - } else if (pTsdb->rCache.uid == uid) { - return 0; } pTsdb->rCache.suid = suid; pTsdb->rCache.uid = uid; + pTsdb->rCache.sver = sver; tDestroyTSchema(pTsdb->rCache.pTSchema); return metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, -1, &pTsdb->rCache.pTSchema); } @@ -1335,36 +1335,27 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 int32_t code = 0, lino = 0; // 1. prepare last - TSDBROW lRow = {.type = TSDBROW_ROW_FMT, .pTSRow = aRow[nRow - 1], .version = version}; - + TSDBROW lRow = {.type = TSDBROW_ROW_FMT, .pTSRow = aRow[nRow - 1], .version = version}; STSchema *pTSchema = NULL; int32_t sver = TSDBROW_SVERSION(&lRow); SArray *ctxArray = NULL; SSHashObj *iColHash = NULL; - TAOS_CHECK_GOTO(tsdbUpdateSkm(pTsdb, suid, uid), &lino, _exit); + TAOS_CHECK_GOTO(tsdbUpdateSkm(pTsdb, suid, uid, sver), &lino, _exit); pTSchema = pTsdb->rCache.pTSchema; - /* - TAOS_CHECK_GOTO(metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema), &lino, _exit); - */ TSDBROW tRow = {.type = TSDBROW_ROW_FMT, .version = version}; int32_t nCol = pTSchema->numOfCols; - ctxArray = taosArrayInit(nCol, sizeof(SLastUpdateCtx)); - iColHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)); + ctxArray = taosArrayInit(nCol * 2, sizeof(SLastUpdateCtx)); // 1. prepare by lrow STsdbRowKey tsdbRowKey = {0}; tsdbRowGetKey(&lRow, &tsdbRowKey); STSDBRowIter iter = {0}; - code = tsdbRowIterOpen(&iter, &lRow, pTSchema); - if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s tsdbRowIterOpen failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - tstrerror(code)); - TAOS_CHECK_GOTO(code, &lino, _exit); - } + TAOS_CHECK_GOTO(tsdbRowIterOpen(&iter, &lRow, pTSchema), &lino, _exit); + int32_t iCol = 0; for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal && iCol < nCol; pColVal = tsdbRowIterNext(&iter), iCol++) { SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST_ROW, .tsdbRowKey = tsdbRowKey, .colVal = *pColVal}; @@ -1372,15 +1363,19 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 TAOS_CHECK_GOTO(terrno, &lino, _exit); } - if (!COL_VAL_IS_VALUE(pColVal)) { + if (COL_VAL_IS_VALUE(pColVal)) { + updateCtx.lflag = LFLAG_LAST; + if (!taosArrayPush(ctxArray, &updateCtx)) { + TAOS_CHECK_GOTO(terrno, &lino, _exit); + } + } else { + if (!iColHash) { + iColHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)); + } + if (tSimpleHashPut(iColHash, &iCol, sizeof(iCol), NULL, 0)) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } - continue; - } - updateCtx.lflag = LFLAG_LAST; - if (!taosArrayPush(ctxArray, &updateCtx)) { - TAOS_CHECK_GOTO(terrno, &lino, _exit); } } tsdbRowClose(&iter); @@ -1425,7 +1420,10 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 } _exit: - // taosMemoryFreeClear(pTSchema); + if (code) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); + } + taosArrayDestroy(ctxArray); tSimpleHashCleanup(iColHash); diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index 92dcf015a6..160808f956 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -14,12 +14,12 @@ */ #define _DEFAULT_SOURCE +#include "tlrucache.h" #include "os.h" #include "taoserror.h" #include "tarray.h" #include "tdef.h" #include "tlog.h" -#include "tlrucache.h" #include "tutil.h" typedef struct SLRUEntry SLRUEntry; @@ -110,7 +110,7 @@ struct SLRUEntryTable { }; static int taosLRUEntryTableInit(SLRUEntryTable *table, int maxUpperHashBits) { - table->lengthBits = 4; + table->lengthBits = 16; table->list = taosMemoryCalloc(1 << table->lengthBits, sizeof(SLRUEntry *)); if (!table->list) { TAOS_RETURN(terrno); @@ -168,7 +168,14 @@ static SLRUEntry **taosLRUEntryTableFindPtr(SLRUEntryTable *table, const void *k while (*entry && ((*entry)->hash != hash || memcmp(key, (*entry)->keyData, keyLen) != 0)) { entry = &(*entry)->nextHash; } - + /* + SLRUEntry *pentry = table->list[hash >> (32 - table->lengthBits)]; + SLRUEntry **entry = &table->list[hash >> (32 - table->lengthBits)]; + while (pentry && (pentry->hash != hash || memcmp(key, pentry->keyData, keyLen) != 0)) { + entry = &pentry->nextHash; + pentry = *entry; + } + */ return entry; } @@ -371,9 +378,9 @@ static void taosLRUCacheShardCleanup(SLRUCacheShard *shard) { static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry *e, LRUHandle **handle, bool freeOnFail) { - LRUStatus status = TAOS_LRU_STATUS_OK; + LRUStatus status = TAOS_LRU_STATUS_OK; SLRUEntry *toFree = NULL; - SArray *lastReferenceList = NULL; + SArray *lastReferenceList = NULL; if (shard->usage + e->totalCharge > shard->capacity) { lastReferenceList = taosArrayInit(16, POINTER_BYTES); if (!lastReferenceList) { @@ -744,7 +751,8 @@ void taosLRUCacheCleanup(SLRUCache *cache) { } LRUStatus taosLRUCacheInsert(SLRUCache *cache, const void *key, size_t keyLen, void *value, size_t charge, - _taos_lru_deleter_t deleter, _taos_lru_overwriter_t overwriter, LRUHandle **handle, LRUPriority priority, void *ud) { + _taos_lru_deleter_t deleter, _taos_lru_overwriter_t overwriter, LRUHandle **handle, + LRUPriority priority, void *ud) { uint32_t hash = TAOS_LRU_CACHE_SHARD_HASH32(key, keyLen); uint32_t shardIndex = hash & cache->shardedCache.shardMask; From d8df6db06ab039e47a6356b3870e15454d5d83e9 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 30 Oct 2024 10:22:19 +0800 Subject: [PATCH 23/34] fix schema update with sver param --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/meta/metaTable.c | 4 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 76 ++++++++++++------------- source/util/src/tlrucache.c | 20 +++++-- 5 files changed, 55 insertions(+), 48 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index f1ee5267a1..c1123db7a3 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -343,6 +343,7 @@ typedef struct { rocksdb_readoptions_t *readoptions; rocksdb_writebatch_t *writebatch; TdThreadMutex writeBatchMutex; + int32_t sver; tb_uid_t suid; tb_uid_t uid; STSchema *pTSchema; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 7673e562a8..98f11086bc 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -222,7 +222,7 @@ int32_t tsdbCacheNewSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, int8_t int32_t tsdbCacheDropSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, bool hasPrimayKey); int32_t tsdbCacheNewNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, int8_t col_type); int32_t tsdbCacheDropNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, bool hasPrimayKey); -void tsdbCacheInvalidateSchema(STsdb* pTsdb, tb_uid_t suid, tb_uid_t uid); +void tsdbCacheInvalidateSchema(STsdb* pTsdb, tb_uid_t suid, tb_uid_t uid, int32_t sver); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index f525a28bd6..5c3516a962 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -621,7 +621,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { } if (uids) taosArrayDestroy(uids); - tsdbCacheInvalidateSchema(pTsdb, pReq->suid, -1); + tsdbCacheInvalidateSchema(pTsdb, pReq->suid, -1, pReq->schemaRow.version); } metaWLock(pMeta); @@ -1948,7 +1948,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl } if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, entry.uid); + tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, entry.uid, pSchema->version); } entry.version = version; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 801e89c2b6..16812b16f4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1136,19 +1136,17 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray (void)taosThreadMutexLock(&pTsdb->lruMutex); for (int i = 0; i < num_keys; ++i) { - SLastUpdateCtx *updCtx = (SLastUpdateCtx *)taosArrayGet(updCtxArray, i); - - int8_t lflag = updCtx->lflag; - SRowKey *pRowKey = &updCtx->tsdbRowKey.key; - SColVal *pColVal = &updCtx->colVal; + SLastUpdateCtx *updCtx = &((SLastUpdateCtx *)TARRAY_DATA(updCtxArray))[i]; + int8_t lflag = updCtx->lflag; + SRowKey *pRowKey = &updCtx->tsdbRowKey.key; + SColVal *pColVal = &updCtx->colVal; if (lflag == LFLAG_LAST && !COL_VAL_IS_VALUE(pColVal)) { continue; } SLastKey *key = &(SLastKey){.lflag = lflag, .uid = uid, .cid = pColVal->cid}; - size_t klen = ROCKS_KEY_LEN; - LRUHandle *h = taosLRUCacheLookup(pCache, key, klen); + LRUHandle *h = taosLRUCacheLookup(pCache, key, ROCKS_KEY_LEN); if (h) { SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h); if (pLastCol->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) { @@ -1304,28 +1302,30 @@ _exit: TAOS_RETURN(code); } -void tsdbCacheInvalidateSchema(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid) { - if (suid) { - if (pTsdb->rCache.suid == suid) { +void tsdbCacheInvalidateSchema(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int32_t sver) { + if (pTsdb->rCache.pTSchema && pTsdb->rCache.suid == suid) { + if (pTsdb->rCache.suid && sver == pTsdb->rCache.sver) { + pTsdb->rCache.sver = -1; pTsdb->rCache.suid = -1; + } else if (pTsdb->rCache.uid == uid && pTsdb->rCache.sver == sver) { + pTsdb->rCache.sver = -1; + pTsdb->rCache.uid = -1; } - } else if (pTsdb->rCache.uid == uid) { - pTsdb->rCache.uid = -1; } } -static int32_t tsdbUpdateSkm(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid) { - if (suid) { - if (pTsdb->rCache.suid == suid) { - pTsdb->rCache.uid = uid; +static int32_t tsdbUpdateSkm(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int32_t sver) { + if (pTsdb->rCache.pTSchema && pTsdb->rCache.suid == suid) { + if (pTsdb->rCache.suid && sver == pTsdb->rCache.sver) { + return 0; + } else if (pTsdb->rCache.uid == uid && pTsdb->rCache.sver == sver) { return 0; } - } else if (pTsdb->rCache.uid == uid) { - return 0; } pTsdb->rCache.suid = suid; pTsdb->rCache.uid = uid; + pTsdb->rCache.sver = sver; tDestroyTSchema(pTsdb->rCache.pTSchema); return metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, -1, &pTsdb->rCache.pTSchema); } @@ -1335,36 +1335,27 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 int32_t code = 0, lino = 0; // 1. prepare last - TSDBROW lRow = {.type = TSDBROW_ROW_FMT, .pTSRow = aRow[nRow - 1], .version = version}; - + TSDBROW lRow = {.type = TSDBROW_ROW_FMT, .pTSRow = aRow[nRow - 1], .version = version}; STSchema *pTSchema = NULL; int32_t sver = TSDBROW_SVERSION(&lRow); SArray *ctxArray = NULL; SSHashObj *iColHash = NULL; - TAOS_CHECK_GOTO(tsdbUpdateSkm(pTsdb, suid, uid), &lino, _exit); + TAOS_CHECK_GOTO(tsdbUpdateSkm(pTsdb, suid, uid, sver), &lino, _exit); pTSchema = pTsdb->rCache.pTSchema; - /* - TAOS_CHECK_GOTO(metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema), &lino, _exit); - */ TSDBROW tRow = {.type = TSDBROW_ROW_FMT, .version = version}; int32_t nCol = pTSchema->numOfCols; - ctxArray = taosArrayInit(nCol, sizeof(SLastUpdateCtx)); - iColHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)); + ctxArray = taosArrayInit(nCol * 2, sizeof(SLastUpdateCtx)); // 1. prepare by lrow STsdbRowKey tsdbRowKey = {0}; tsdbRowGetKey(&lRow, &tsdbRowKey); STSDBRowIter iter = {0}; - code = tsdbRowIterOpen(&iter, &lRow, pTSchema); - if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d, %s tsdbRowIterOpen failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - tstrerror(code)); - TAOS_CHECK_GOTO(code, &lino, _exit); - } + TAOS_CHECK_GOTO(tsdbRowIterOpen(&iter, &lRow, pTSchema), &lino, _exit); + int32_t iCol = 0; for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal && iCol < nCol; pColVal = tsdbRowIterNext(&iter), iCol++) { SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST_ROW, .tsdbRowKey = tsdbRowKey, .colVal = *pColVal}; @@ -1372,15 +1363,19 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 TAOS_CHECK_GOTO(terrno, &lino, _exit); } - if (!COL_VAL_IS_VALUE(pColVal)) { + if (COL_VAL_IS_VALUE(pColVal)) { + updateCtx.lflag = LFLAG_LAST; + if (!taosArrayPush(ctxArray, &updateCtx)) { + TAOS_CHECK_GOTO(terrno, &lino, _exit); + } + } else { + if (!iColHash) { + iColHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)); + } + if (tSimpleHashPut(iColHash, &iCol, sizeof(iCol), NULL, 0)) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } - continue; - } - updateCtx.lflag = LFLAG_LAST; - if (!taosArrayPush(ctxArray, &updateCtx)) { - TAOS_CHECK_GOTO(terrno, &lino, _exit); } } tsdbRowClose(&iter); @@ -1425,7 +1420,10 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 } _exit: - // taosMemoryFreeClear(pTSchema); + if (code) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); + } + taosArrayDestroy(ctxArray); tSimpleHashCleanup(iColHash); diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index 92dcf015a6..160808f956 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -14,12 +14,12 @@ */ #define _DEFAULT_SOURCE +#include "tlrucache.h" #include "os.h" #include "taoserror.h" #include "tarray.h" #include "tdef.h" #include "tlog.h" -#include "tlrucache.h" #include "tutil.h" typedef struct SLRUEntry SLRUEntry; @@ -110,7 +110,7 @@ struct SLRUEntryTable { }; static int taosLRUEntryTableInit(SLRUEntryTable *table, int maxUpperHashBits) { - table->lengthBits = 4; + table->lengthBits = 16; table->list = taosMemoryCalloc(1 << table->lengthBits, sizeof(SLRUEntry *)); if (!table->list) { TAOS_RETURN(terrno); @@ -168,7 +168,14 @@ static SLRUEntry **taosLRUEntryTableFindPtr(SLRUEntryTable *table, const void *k while (*entry && ((*entry)->hash != hash || memcmp(key, (*entry)->keyData, keyLen) != 0)) { entry = &(*entry)->nextHash; } - + /* + SLRUEntry *pentry = table->list[hash >> (32 - table->lengthBits)]; + SLRUEntry **entry = &table->list[hash >> (32 - table->lengthBits)]; + while (pentry && (pentry->hash != hash || memcmp(key, pentry->keyData, keyLen) != 0)) { + entry = &pentry->nextHash; + pentry = *entry; + } + */ return entry; } @@ -371,9 +378,9 @@ static void taosLRUCacheShardCleanup(SLRUCacheShard *shard) { static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry *e, LRUHandle **handle, bool freeOnFail) { - LRUStatus status = TAOS_LRU_STATUS_OK; + LRUStatus status = TAOS_LRU_STATUS_OK; SLRUEntry *toFree = NULL; - SArray *lastReferenceList = NULL; + SArray *lastReferenceList = NULL; if (shard->usage + e->totalCharge > shard->capacity) { lastReferenceList = taosArrayInit(16, POINTER_BYTES); if (!lastReferenceList) { @@ -744,7 +751,8 @@ void taosLRUCacheCleanup(SLRUCache *cache) { } LRUStatus taosLRUCacheInsert(SLRUCache *cache, const void *key, size_t keyLen, void *value, size_t charge, - _taos_lru_deleter_t deleter, _taos_lru_overwriter_t overwriter, LRUHandle **handle, LRUPriority priority, void *ud) { + _taos_lru_deleter_t deleter, _taos_lru_overwriter_t overwriter, LRUHandle **handle, + LRUPriority priority, void *ud) { uint32_t hash = TAOS_LRU_CACHE_SHARD_HASH32(key, keyLen); uint32_t shardIndex = hash & cache->shardedCache.shardMask; From e6e36bcd2dd0c53d271105189ff3e8d720d94a2a Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 29 Oct 2024 12:17:14 +0800 Subject: [PATCH 24/34] fix PARITION (INTERVAL) HAVING() caused planner internal err --- source/libs/parser/src/parTranslater.c | 2 +- tests/system-test/2-query/fill_with_group.py | 32 ++++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 395801e3dd..34b8290b67 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3752,7 +3752,7 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { bool partionByTbname = hasTbnameFunction(pSelect->pPartitionByList); FOREACH(pPartKey, pSelect->pPartitionByList) { if (nodesEqualNode(pPartKey, *pNode)) { - return pCxt->currClause == SQL_CLAUSE_HAVING ? DEAL_RES_IGNORE_CHILD : rewriteExprToGroupKeyFunc(pCxt, pNode); + return pSelect->hasAggFuncs ? rewriteExprToGroupKeyFunc(pCxt, pNode) : DEAL_RES_IGNORE_CHILD; } if ((partionByTbname) && QUERY_NODE_COLUMN == nodeType(*pNode) && ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) { diff --git a/tests/system-test/2-query/fill_with_group.py b/tests/system-test/2-query/fill_with_group.py index 3b98ec30ce..4a40338504 100644 --- a/tests/system-test/2-query/fill_with_group.py +++ b/tests/system-test/2-query/fill_with_group.py @@ -344,9 +344,35 @@ class TDTestCase: tdSql.query(sql, queryTimes=1) tdSql.checkRows(55) - # TODO Fix Me! - sql = "explain SELECT count(*), timediff(_wend, last(ts)), timediff('2018-09-20 01:00:00', _wstart) FROM meters WHERE ts >= '2018-09-20 00:00:00.000' AND ts < '2018-09-20 01:00:00.000' PARTITION BY concat(tbname, 'asd') INTERVAL(5m) having(concat(tbname, 'asd') like '%asd');" - tdSql.error(sql, -2147473664) # Error: Planner internal error + sql = "SELECT count(*), timediff(_wend, last(ts)), timediff('2018-09-20 01:00:00', _wstart) FROM meters WHERE ts >= '2018-09-20 00:00:00.000' AND ts < '2018-09-20 01:00:00.000' PARTITION BY concat(tbname, 'asd') INTERVAL(5m) having(concat(tbname, 'asd') like '%asd');" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(60) + + sql = "SELECT count(*), timediff(_wend, last(ts)), timediff('2018-09-20 01:00:00', _wstart) FROM meters WHERE ts >= '2018-09-20 00:00:00.000' AND ts < '2018-09-20 01:00:00.000' PARTITION BY concat(tbname, 'asd') INTERVAL(5m) having(concat(tbname, 'asd') like 'asd%');" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(0) + + sql = "SELECT c1 FROM meters PARTITION BY c1 HAVING c1 > 0 slimit 2 limit 10" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(20) + + sql = "SELECT t1 FROM meters PARTITION BY t1 HAVING(t1 = 1)" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(20000) + + sql = "SELECT concat(t2, 'asd') FROM meters PARTITION BY t2 HAVING(t2 like '%5')" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(10000) + tdSql.checkData(0, 0, 'tb5asd') + + sql = "SELECT concat(t2, 'asd') FROM meters PARTITION BY concat(t2, 'asd') HAVING(concat(t2, 'asd')like '%5%')" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(10000) + tdSql.checkData(0, 0, 'tb5asd') + + sql = "SELECT avg(c1) FROM meters PARTITION BY tbname, t1 HAVING(t1 = 1)" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(2) def run(self): self.prepareTestEnv() From 14ddaf7c3932216f5e7c0366ad3498eba6c1baab Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Wed, 30 Oct 2024 11:08:41 +0800 Subject: [PATCH 25/34] add parition having tests and fix merge align interval + limit --- source/libs/executor/src/timewindowoperator.c | 7 +++- tests/system-test/2-query/fill_with_group.py | 37 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 34ecda6ce7..8c5f628034 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2018,7 +2018,12 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { cleanupAfterGroupResultGen(pMiaInfo, pRes); code = doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL); QUERY_CHECK_CODE(code, lino, _end); - break; + if (pRes->info.rows == 0) { + // After filtering for last group, the result is empty, so we need to continue to process next group + continue; + } else { + break; + } } else { // continue pRes->info.id.groupId = pMiaInfo->groupId; diff --git a/tests/system-test/2-query/fill_with_group.py b/tests/system-test/2-query/fill_with_group.py index 4a40338504..3847f1dec4 100644 --- a/tests/system-test/2-query/fill_with_group.py +++ b/tests/system-test/2-query/fill_with_group.py @@ -374,6 +374,43 @@ class TDTestCase: tdSql.query(sql, queryTimes=1) tdSql.checkRows(2) + sql = "SELECT count(*) FROM meters PARTITION BY concat(tbname, 'asd') HAVING(concat(tbname, 'asd') like '%asd')" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(10) + + sql = "SELECT count(*), concat(tbname, 'asd') FROM meters PARTITION BY concat(tbname, 'asd') HAVING(concat(tbname, 'asd') like '%asd')" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(10) + + sql = "SELECT count(*) FROM meters PARTITION BY t1 HAVING(t1 < 4) order by t1 +1" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(4) + + sql = "SELECT count(*), t1 + 100 FROM meters PARTITION BY t1 HAVING(t1 < 4) order by t1 +1" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(4) + + sql = "SELECT count(*), t1 + 100 FROM meters PARTITION BY t1 INTERVAL(1d) HAVING(t1 < 4) order by t1 +1 desc" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(280) + + sql = "SELECT count(*), concat(t3, 'asd') FROM meters PARTITION BY concat(t3, 'asd') INTERVAL(1d) HAVING(concat(t3, 'asd') like '%5asd' and count(*) = 118)" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(1) + + sql = "SELECT count(*), concat(t3, 'asd') FROM meters PARTITION BY concat(t3, 'asd') INTERVAL(1d) HAVING(concat(t3, 'asd') like '%5asd' and count(*) != 118)" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(69) + + sql = "SELECT count(*), concat(t3, 'asd') FROM meters PARTITION BY concat(t3, 'asd') INTERVAL(1d) HAVING(concat(t3, 'asd') like '%5asd') order by count(*) asc limit 10" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(10) + + sql = "SELECT count(*), concat(t3, 'asd') FROM meters PARTITION BY concat(t3, 'asd') INTERVAL(1d) HAVING(concat(t3, 'asd') like '%5asd' or concat(t3, 'asd') like '%3asd') order by count(*) asc limit 10000" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(140) + + def run(self): self.prepareTestEnv() self.test_partition_by_with_interval_fill_prev_new_group_fill_error() From ba88253bf19be9a21855eb8e8a7edf0d012bb7ca Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 30 Oct 2024 14:42:13 +0800 Subject: [PATCH 26/34] lru: remove commented section --- source/util/src/tlrucache.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index 160808f956..43a220af50 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -168,14 +168,7 @@ static SLRUEntry **taosLRUEntryTableFindPtr(SLRUEntryTable *table, const void *k while (*entry && ((*entry)->hash != hash || memcmp(key, (*entry)->keyData, keyLen) != 0)) { entry = &(*entry)->nextHash; } - /* - SLRUEntry *pentry = table->list[hash >> (32 - table->lengthBits)]; - SLRUEntry **entry = &table->list[hash >> (32 - table->lengthBits)]; - while (pentry && (pentry->hash != hash || memcmp(key, pentry->keyData, keyLen) != 0)) { - entry = &pentry->nextHash; - pentry = *entry; - } - */ + return entry; } From 6da22e64838a677c7bd68e8d017771963fb0a01e Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 30 Oct 2024 16:13:27 +0800 Subject: [PATCH 27/34] enh: tsdb cache schema --- source/dnode/vnode/src/tsdb/tsdbCache.c | 37 ++++++++++++++----------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 16812b16f4..334725d9d0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -230,6 +230,7 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { pTsdb->rCache.readoptions = readoptions; pTsdb->rCache.flushoptions = flushoptions; pTsdb->rCache.db = db; + pTsdb->rCache.sver = -1; pTsdb->rCache.suid = -1; pTsdb->rCache.uid = -1; pTsdb->rCache.pTSchema = NULL; @@ -1303,31 +1304,35 @@ _exit: } void tsdbCacheInvalidateSchema(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int32_t sver) { - if (pTsdb->rCache.pTSchema && pTsdb->rCache.suid == suid) { - if (pTsdb->rCache.suid && sver == pTsdb->rCache.sver) { - pTsdb->rCache.sver = -1; - pTsdb->rCache.suid = -1; - } else if (pTsdb->rCache.uid == uid && pTsdb->rCache.sver == sver) { - pTsdb->rCache.sver = -1; - pTsdb->rCache.uid = -1; - } + SRocksCache *pRCache = &pTsdb->rCache; + if (!pRCache->pTSchema || sver <= pTsdb->rCache.sver) return; + + if (suid > 0 && suid == pRCache->suid) { + pRCache->sver = -1; + pRCache->suid = -1; + } + if (suid == 0 && uid == pRCache->uid) { + pRCache->sver = -1; + pRCache->uid = -1; } } static int32_t tsdbUpdateSkm(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int32_t sver) { - if (pTsdb->rCache.pTSchema && pTsdb->rCache.suid == suid) { - if (pTsdb->rCache.suid && sver == pTsdb->rCache.sver) { + SRocksCache *pRCache = &pTsdb->rCache; + if (pRCache->pTSchema && sver == pRCache->sver) { + if (suid > 0 && suid == pRCache->suid) { return 0; - } else if (pTsdb->rCache.uid == uid && pTsdb->rCache.sver == sver) { + } + if (suid == 0 && uid == pRCache->uid) { return 0; } } - pTsdb->rCache.suid = suid; - pTsdb->rCache.uid = uid; - pTsdb->rCache.sver = sver; - tDestroyTSchema(pTsdb->rCache.pTSchema); - return metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, -1, &pTsdb->rCache.pTSchema); + pRCache->suid = suid; + pRCache->uid = uid; + pRCache->sver = sver; + tDestroyTSchema(pRCache->pTSchema); + return metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, -1, &pRCache->pTSchema); } int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int64_t version, int32_t nRow, From d70db52bb953ae2bf83c72af9ccf708d5e62e031 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 31 Oct 2024 09:29:05 +0800 Subject: [PATCH 28/34] Update 14-stream.md --- docs/zh/14-reference/03-taos-sql/14-stream.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/zh/14-reference/03-taos-sql/14-stream.md b/docs/zh/14-reference/03-taos-sql/14-stream.md index cd5c76a4ad..9567381f7d 100644 --- a/docs/zh/14-reference/03-taos-sql/14-stream.md +++ b/docs/zh/14-reference/03-taos-sql/14-stream.md @@ -291,3 +291,4 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; CREATE SNODE ON DNODE [id] ``` 其中的 id 是集群中的 dnode 的序号。请注意选择的dnode，流计算的中间状态将自动在其上进行备份。 +从 3.3.4.0 版本开始，在多副本环境中创建流会进行 snode 的**存在性检查**，要求首先创建 snode。如果 snode 不存在，无法创建流。 From 2d310221b34327b2d702c56b11e76409f67484f2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 31 Oct 2024 10:03:43 +0800 Subject: [PATCH 29/34] enh:[TD-32166]refactor code in sml --- source/client/src/clientSml.c | 72 +++++++++++++------------------ source/client/src/clientSmlJson.c | 3 +- 2 files changed, 30 insertions(+), 45 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 9d3b28e105..e8221c8b8d 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -138,7 +138,7 @@ void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2 if (pBuf->buf == NULL) { return; } - (void)memset(pBuf->buf, 0, pBuf->len); + pBuf->buf[0] = 0; if (msg1) { (void)strncat(pBuf->buf, msg1, pBuf->len - 1); } @@ -475,7 +475,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam } } if (autoChildName) { - (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + childTableName[0] = '\0'; for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); SML_CHECK_NULL(tag); @@ -499,7 +499,6 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnam SML_CHECK_NULL(tag); // handle child table name if (childTableNameLen == tag->keyLen && strncmp(tag->key, tbnameKey, tag->keyLen) == 0) { - (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); tstrncpy(childTableName, tag->value, TMIN(TSDB_TABLE_NAME_LEN, tag->length + 1)); if (tsSmlDot2Underline) { smlStrReplace(childTableName, strlen(childTableName)); @@ -524,8 +523,7 @@ int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { dst = taosArrayDup(oneTable->tags, NULL); SML_CHECK_NULL(dst); if (oneTable->sTableNameLen >= TSDB_TABLE_NAME_LEN) { - code = TSDB_CODE_SML_INTERNAL_ERROR; - goto END; + SML_CHECK_CODE(TSDB_CODE_SML_INTERNAL_ERROR); } char superName[TSDB_TABLE_NAME_LEN] = {0}; RandTableName rName = {dst, NULL, (uint8_t)oneTable->sTableNameLen, oneTable->childTableName}; @@ -676,8 +674,9 @@ int32_t smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen, ST conn.requestId = info->pRequest->requestId; conn.requestObjRefId = info->pRequest->self; conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + int32_t len = TMIN(measureLen, TSDB_TABLE_NAME_LEN - 1); (void)memcpy(pName.tname, measure, measureLen); + pName.tname[len] = 0; return catalogGetSTableMeta(info->pCatalog, &conn, &pName, pTableMeta); } @@ -766,8 +765,7 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH SSmlKv *kv = (SSmlKv *)taosArrayGet(checkDumplicateCols, j); SML_CHECK_NULL(kv); if (taosHashGet(schemaHash, kv->key, kv->keyLen) != NULL) { - code = TSDB_CODE_PAR_DUPLICATED_COLUMN; - goto END; + SML_CHECK_CODE(TSDB_CODE_PAR_DUPLICATED_COLUMN); } } END: @@ -788,8 +786,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); SML_CHECK_NULL(kv); if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { - code = TSDB_CODE_SML_INVALID_DATA; - goto END; + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } } @@ -810,7 +807,7 @@ static int32_t getBytes(uint8_t type, int32_t length) { static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray *results, int32_t numOfCols, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = TSDB_CODE_SUCCESS; + int32_t lino = 0; for (int j = 0; j < taosArrayGetSize(cols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); SML_CHECK_NULL(kv); @@ -825,9 +822,7 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); if (index == NULL) { - uError("smlBuildFieldsList get error, key:%s", kv->key); - code = TSDB_CODE_SML_INVALID_DATA; - goto END; + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } uint16_t newIndex = *index; if (isTag) newIndex -= numOfCols; @@ -894,17 +889,14 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pSql = (action == SCHEMA_ACTION_ADD_COLUMN) ? "sml_add_column" : "sml_modify_column_size"; smlBuildCreateStbReq(&pReq, pTableMeta->sversion + 1, pTableMeta->tversion, pTableMeta->uid, TD_REQ_FROM_TAOX); } else { - uError("SML:0x%" PRIx64 " invalid action:%d", info->id, action); - code = TSDB_CODE_SML_INVALID_DATA; - goto END; + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } SML_CHECK_CODE(buildRequest(info->taos->id, pSql, strlen(pSql), NULL, false, &pRequest, 0)); pRequest->syncQuery = true; if (!pRequest->pDb) { - code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto END; + SML_CHECK_CODE(TSDB_CODE_PAR_DB_NOT_SPECIFIED); } if (pReq.numOfTags == 0) { @@ -924,15 +916,16 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pCmdMsg.msgType = TDMT_MND_CREATE_STB; pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); if (pCmdMsg.msgLen < 0) { - code = pCmdMsg.msgLen; - goto END; + uError("failed to serialize create stable request1, code:%d, terrno:%d", pCmdMsg.msgLen, terrno); + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); SML_CHECK_NULL(pCmdMsg.pMsg); code = tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); if (code < 0) { taosMemoryFree(pCmdMsg.pMsg); - goto END; + uError("failed to serialize create stable request2, code:%d, terrno:%d", code, terrno); + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } SQuery pQuery = {0}; @@ -1108,8 +1101,9 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen); } smlStrReplace(measure, superTableLen); - (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - (void)memcpy(pName.tname, measure, TMIN(superTableLen, TSDB_TABLE_NAME_LEN - 1)); + size_t nameLen = TMIN(superTableLen, TSDB_TABLE_NAME_LEN - 1); + (void)memcpy(pName.tname, measure, nameLen); + pName.tname[nameLen] = '\0'; taosMemoryFree(measure); code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); @@ -1118,8 +1112,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { SML_CHECK_CODE(smlCreateTable(info, &conn, sTableData, &pName, &pTableMeta)); } else if (code == TSDB_CODE_SUCCESS) { if (smlIsPKTable(pTableMeta)) { - code = TSDB_CODE_SML_NOT_SUPPORT_PK; - goto END; + SML_CHECK_CODE(TSDB_CODE_SML_NOT_SUPPORT_PK); } hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); @@ -1176,8 +1169,7 @@ static int32_t smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols if (ret == 0) { SML_CHECK_NULL(taosArrayPush(metaArray, kv)); if (taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { - code = TSDB_CODE_PAR_DUPLICATED_COLUMN; - goto END; + SML_CHECK_CODE(TSDB_CODE_PAR_DUPLICATED_COLUMN); } } else if (terrno == TSDB_CODE_DUP_KEY) { return TSDB_CODE_PAR_DUPLICATED_COLUMN; @@ -1208,8 +1200,7 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols } if (kv->type != value->type) { smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); - code = TSDB_CODE_SML_NOT_SAME_TYPE; - goto END; + SML_CHECK_CODE(TSDB_CODE_SML_NOT_SAME_TYPE); } if (IS_VAR_DATA_TYPE(kv->type) && (kv->length > value->length)) { // update string len, if bigger @@ -1219,16 +1210,13 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols size_t tmp = taosArrayGetSize(metaArray); if (tmp > INT16_MAX) { smlBuildInvalidDataMsg(msg, "too many cols or tags", kv->key); - uError("too many cols or tags"); - code = TSDB_CODE_SML_INVALID_DATA; - goto END; + SML_CHECK_CODE(TSDB_CODE_SML_INVALID_DATA); } int16_t size = tmp; SML_CHECK_CODE(taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES)); SML_CHECK_NULL(taosArrayPush(metaArray, kv)); if (taosHashGet(checkDuplicate, kv->key, kv->keyLen) != NULL) { - code = TSDB_CODE_PAR_DUPLICATED_COLUMN; - goto END; + SML_CHECK_CODE(TSDB_CODE_PAR_DUPLICATED_COLUMN); } } } @@ -1349,8 +1337,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { terrno = 0; code = taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { - code = TSDB_CODE_PAR_DUPLICATED_COLUMN; - goto END; + SML_CHECK_CODE(TSDB_CODE_PAR_DUPLICATED_COLUMN); } SML_CHECK_CODE(code); } @@ -1417,8 +1404,7 @@ static int32_t smlParseEnd(SSmlHandle *info) { code = taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); if (code != TSDB_CODE_SUCCESS) { smlDestroySTableMeta(&meta); - uError("SML:0x%" PRIx64 " put measure to hash failed", info->id); - goto END; + SML_CHECK_CODE(code); } SML_CHECK_CODE(smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags, NULL)); SML_CHECK_CODE(smlInsertMeta(meta->colHash, meta->cols, elements->colArray, meta->tagHash)); @@ -1459,8 +1445,8 @@ static int32_t smlInsertData(SSmlHandle *info) { PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); } smlStrReplace(measure, measureLen); - (void)memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); (void)memcpy(pName.tname, measure, measureLen); + pName.tname[measureLen] = '\0'; if (info->pRequest->tableList == NULL) { info->pRequest->tableList = taosArrayInit(1, sizeof(SName)); @@ -1485,8 +1471,7 @@ static int32_t smlInsertData(SSmlHandle *info) { (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); if (unlikely(NULL == pMeta || NULL == *pMeta || NULL == (*pMeta)->tableMeta)) { uError("SML:0x%" PRIx64 " %s NULL == pMeta. table name: %s", info->id, __FUNCTION__, tableData->childTableName); - code = TSDB_CODE_SML_INTERNAL_ERROR; - goto END; + SML_CHECK_CODE(TSDB_CODE_SML_INTERNAL_ERROR); } // use tablemeta of stable to save vgid and uid of child table @@ -1565,12 +1550,13 @@ END: } static void printRaw(int64_t id, int lineNum, int numLines, ELogLevel level, char* data, int32_t len){ - char *print = taosMemoryCalloc(len + 1, 1); + char *print = taosMemoryMalloc(len + 1); if (print == NULL) { uError("SML:0x%" PRIx64 " smlParseLine failed. code : %d", id, terrno); return; } (void)memcpy(print, data, len); + print[len] = '\0'; if (level == DEBUG_DEBUG){ uDebug("SML:0x%" PRIx64 " smlParseLine is raw, line %d/%d : %s", id, lineNum, numLines, print); }else if (level == DEBUG_ERROR){ diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 2dcc36eada..9a27a30d84 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -470,8 +470,7 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo uError("SML:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts); taosMemoryFree(tmp); } - code = TSDB_CODE_INVALID_TIMESTAMP; - goto END; + SML_CHECK_CODE(TSDB_CODE_INVALID_TIMESTAMP); } SSmlKv kvTs = {0}; smlBuildTsKv(&kvTs, ts); From 40cf21b15ce07e56a0c2d43b4567675ddf852b82 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 31 Oct 2024 11:24:31 +0800 Subject: [PATCH 30/34] fix firstVer incorrect while romove wal dir --- source/libs/wal/src/walWrite.c | 2 +- tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index c8c37b11bc..1c53baf360 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -664,7 +664,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy // set status if (pWal->vers.firstVer == -1) { - pWal->vers.firstVer = 0; + pWal->vers.firstVer = index; } pWal->vers.lastVer = index; pWal->totSize += sizeof(SWalCkHead) + cyptedBodyLen; diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py index e0632686d4..40879d5c66 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py @@ -17,8 +17,6 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: - - updatecfgDict = {'sDebugFlag':143, 'wDebugFlag':143} def __init__(self): self.vgroups = 1 self.ctbNum = 10 From fde2a56076be942bfef4df63b9813240e9c05ba9 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 31 Oct 2024 11:47:32 +0800 Subject: [PATCH 31/34] enh: tsdbCacheRowFormatUpdate check for memory funcs --- source/dnode/vnode/src/tsdb/tsdbCache.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 8e4a53845a..1ae7dfeff7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1350,6 +1350,9 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 int32_t nCol = pTSchema->numOfCols; ctxArray = taosArrayInit(nCol * 2, sizeof(SLastUpdateCtx)); + if (ctxArray == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } // 1. prepare by lrow STsdbRowKey tsdbRowKey = {0}; @@ -1362,20 +1365,27 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal && iCol < nCol; pColVal = tsdbRowIterNext(&iter), iCol++) { SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST_ROW, .tsdbRowKey = tsdbRowKey, .colVal = *pColVal}; if (!taosArrayPush(ctxArray, &updateCtx)) { + tsdbRowClose(&iter); TAOS_CHECK_GOTO(terrno, &lino, _exit); } if (COL_VAL_IS_VALUE(pColVal)) { updateCtx.lflag = LFLAG_LAST; if (!taosArrayPush(ctxArray, &updateCtx)) { + tsdbRowClose(&iter); TAOS_CHECK_GOTO(terrno, &lino, _exit); } } else { if (!iColHash) { iColHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)); + if (ctxArray == NULL) { + tsdbRowClose(&iter); + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); + } } if (tSimpleHashPut(iColHash, &iCol, sizeof(iCol), NULL, 0)) { + tsdbRowClose(&iter); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); } } From e260eb7a07d6e6176b9052ed22a27d8f067aa305 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 31 Oct 2024 13:35:42 +0800 Subject: [PATCH 32/34] Revert "enh(stmt2):add api taos_stmt2_get_all_fields" --- include/client/taos.h | 47 ++++++-------- source/client/inc/clientStmt2.h | 1 - source/client/src/clientMain.c | 25 ++------ source/client/src/clientStmt2.c | 86 -------------------------- source/libs/parser/src/parInsertSql.c | 82 ++++++++++-------------- source/libs/parser/src/parInsertStmt.c | 8 +-- tests/script/api/makefile | 2 - tests/script/api/stmt2-get-fields.c | 63 ------------------- 8 files changed, 58 insertions(+), 256 deletions(-) delete mode 100644 tests/script/api/stmt2-get-fields.c diff --git a/include/client/taos.h b/include/client/taos.h index 266e0e192d..80dbe27c47 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -81,13 +81,6 @@ typedef enum { TSDB_SML_TIMESTAMP_NANO_SECONDS, } TSDB_SML_TIMESTAMP_TYPE; -typedef enum TAOS_FIELD_T { - TAOS_FIELD_COL = 1, - TAOS_FIELD_TAG, - TAOS_FIELD_QUERY, - TAOS_FIELD_TBNAME, -} TAOS_FIELD_T; - typedef struct taosField { char name[65]; int8_t type; @@ -102,15 +95,6 @@ typedef struct TAOS_FIELD_E { int32_t bytes; } TAOS_FIELD_E; -typedef struct TAOS_FIELD_ALL { - char name[65]; - int8_t type; - uint8_t precision; - uint8_t scale; - int32_t bytes; - TAOS_FIELD_T field_type; -} TAOS_FIELD_ALL; - #ifdef WINDOWS #define DLL_EXPORT __declspec(dllexport) #else @@ -211,6 +195,13 @@ DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt); typedef void TAOS_STMT2; +typedef enum TAOS_FIELD_T { + TAOS_FIELD_COL = 1, + TAOS_FIELD_TAG, + TAOS_FIELD_QUERY, + TAOS_FIELD_TBNAME, +} TAOS_FIELD_T; + typedef struct TAOS_STMT2_OPTION { int64_t reqid; bool singleStbInsert; @@ -241,9 +232,7 @@ DLL_EXPORT int taos_stmt2_exec(TAOS_STMT2 *stmt, int *affected_rows); DLL_EXPORT int taos_stmt2_close(TAOS_STMT2 *stmt); DLL_EXPORT int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert); DLL_EXPORT int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields); -DLL_EXPORT int taos_stmt2_get_all_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_ALL **fields); DLL_EXPORT void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_E *fields); -DLL_EXPORT void taos_stmt2_free_all_fields(TAOS_STMT2 *stmt, TAOS_FIELD_ALL *fields); DLL_EXPORT TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt); DLL_EXPORT char *taos_stmt2_error(TAOS_STMT2 *stmt); @@ -262,17 +251,17 @@ DLL_EXPORT int64_t taos_affected_rows64(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); -DLL_EXPORT int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); -DLL_EXPORT void taos_stop_query(TAOS_RES *res); -DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); -DLL_EXPORT int taos_is_null_by_column(TAOS_RES *res, int columnIndex, bool result[], int *rows); -DLL_EXPORT bool taos_is_update_query(TAOS_RES *res); -DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); -DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows); -DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData); -DLL_EXPORT int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex); -DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); -DLL_EXPORT void taos_reset_current_db(TAOS *taos); +DLL_EXPORT int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); +DLL_EXPORT void taos_stop_query(TAOS_RES *res); +DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); +DLL_EXPORT int taos_is_null_by_column(TAOS_RES *res, int columnIndex, bool result[], int *rows); +DLL_EXPORT bool taos_is_update_query(TAOS_RES *res); +DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); +DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows); +DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData); +DLL_EXPORT int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex); +DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); +DLL_EXPORT void taos_reset_current_db(TAOS *taos); DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res); DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res); diff --git a/source/client/inc/clientStmt2.h b/source/client/inc/clientStmt2.h index e54057e72f..4e9a09c082 100644 --- a/source/client/inc/clientStmt2.h +++ b/source/client/inc/clientStmt2.h @@ -222,7 +222,6 @@ int stmtSetTbTags2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *tags); int stmtBindBatch2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *bind, int32_t colIdx); int stmtGetTagFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields); int stmtGetColFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields); -int stmtGetColFieldsALL2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_ALL **fields); int stmtGetParamNum2(TAOS_STMT2 *stmt, int *nums); int stmtGetParamTbName(TAOS_STMT2 *stmt, int *nums); int stmtIsInsert2(TAOS_STMT2 *stmt, int *insert); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index f2fc2b8bdc..64631fd754 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -84,7 +84,7 @@ void taos_cleanup(void) { taosCloseRef(id); nodesDestroyAllocatorSet(); - // cleanupAppInfo(); +// cleanupAppInfo(); rpcCleanup(); tscDebug("rpc cleanup"); @@ -388,6 +388,7 @@ void taos_free_result(TAOS_RES *res) { tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp); } taosMemoryFree(pRsp); + } void taos_kill_query(TAOS *taos) { @@ -483,7 +484,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { return taos_print_row_with_size(str, INT32_MAX, row, fields, num_fields); } -int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { +int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields){ int32_t len = 0; for (int i = 0; i < num_fields; ++i) { if (i > 0 && len < size - 1) { @@ -588,7 +589,7 @@ int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD break; } } - if (len < size) { + if (len < size){ str[len] = 0; } @@ -2081,7 +2082,7 @@ int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert) { } int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields) { - if (stmt == NULL || count == NULL) { + if (stmt == NULL || NULL == count) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -2102,28 +2103,12 @@ int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, } } -int taos_stmt2_get_all_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_ALL **fields) { - if (stmt == NULL || count == NULL) { - tscError("NULL parameter for %s", __FUNCTION__); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - - return stmtGetColFieldsALL2(stmt, count, fields); -} - void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_E *fields) { (void)stmt; if (!fields) return; taosMemoryFree(fields); } -DLL_EXPORT void taos_stmt2_free_all_fields(TAOS_STMT2 *stmt, TAOS_FIELD_ALL *fields) { - (void)stmt; - if (!fields) return; - taosMemoryFree(fields); -} - TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt) { if (stmt == NULL) { tscError("NULL parameter for %s", __FUNCTION__); diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index 67de878ee5..2f046b61d6 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1068,48 +1068,6 @@ static int stmtFetchColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_E return TSDB_CODE_SUCCESS; } - -static int stmtFetchFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_ALL** fields) { - SBoundColInfo* tags = (SBoundColInfo*)pStmt->bInfo.boundTags; - STableMeta* meta = ((SVnodeModifyOpStmt*)(pStmt->sql.pQuery->pRoot))->pTableMeta; - if (tags == NULL || meta == NULL || (meta->schema == NULL && tags->numOfBound != 0)) { - return TSDB_CODE_INVALID_PARA; - } - - if (fields != NULL) { - *fields = taosMemoryCalloc(tags->numOfBound, sizeof(TAOS_FIELD_ALL)); - if (NULL == *fields) { - return terrno; - } - - SSchema* pSchema = meta->schema; - int32_t tbnameIdx = meta->tableInfo.numOfTags + meta->tableInfo.numOfColumns; - for (int32_t i = 0; i < tags->numOfBound; ++i) { - int16_t idx = tags->pColIndex[i]; - if (idx == tbnameIdx) { - (*fields)[i].field_type = TAOS_FIELD_TBNAME; - tstrncpy((*fields)[i].name, "tbname", sizeof((*fields)[i].name)); - continue; - } else if (idx < meta->tableInfo.numOfColumns) { - (*fields)[i].field_type = TAOS_FIELD_COL; - } else { - (*fields)[i].field_type = TAOS_FIELD_TAG; - } - - if (TSDB_DATA_TYPE_TIMESTAMP == pSchema[tags->pColIndex[i]].type) { - (*fields)[i].precision = meta->tableInfo.precision; - } - - tstrncpy((*fields)[i].name, pSchema[tags->pColIndex[i]].name, sizeof((*fields)[i].name)); - (*fields)[i].type = pSchema[tags->pColIndex[i]].type; - (*fields)[i].bytes = pSchema[tags->pColIndex[i]].bytes; - } - } - - *fieldNum = tags->numOfBound; - - return TSDB_CODE_SUCCESS; -} /* SArray* stmtGetFreeCol(STscStmt2* pStmt, int32_t* idx) { while (true) { @@ -1893,50 +1851,6 @@ _return: return code; } -int stmtGetColFieldsALL2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_ALL** fields) { - int32_t code = 0; - STscStmt2* pStmt = (STscStmt2*)stmt; - int32_t preCode = pStmt->errCode; - - STMT_DLOG_E("start to get col fields"); - - if (pStmt->errCode != TSDB_CODE_SUCCESS) { - return pStmt->errCode; - } - - if (STMT_TYPE_QUERY == pStmt->sql.type) { - STMT_ERRI_JRET(TSDB_CODE_TSC_STMT_API_ERROR); - } - - STMT_ERRI_JRET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); - - if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && - STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { - pStmt->bInfo.needParse = false; - } - - if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) { - taos_free_result(pStmt->exec.pRequest); - pStmt->exec.pRequest = NULL; - STMT_ERR_RET(stmtCreateRequest(pStmt)); - } - - STMT_ERRI_JRET(stmtCreateRequest(pStmt)); - - if (pStmt->bInfo.needParse) { - STMT_ERRI_JRET(stmtParseSql(pStmt)); - } - -_return: - - pStmt->errCode = preCode; - if (code == TSDB_CODE_TSC_INVALID_OPERATION) { - return stmtFetchFields2(stmt, nums, fields); - } - - return code; -} - int stmtGetParamNum2(TAOS_STMT2* stmt, int* nums) { STscStmt2* pStmt = (STscStmt2*)stmt; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index f4ef099515..1c26a7c70e 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -30,7 +30,7 @@ typedef struct SInsertParseContext { bool forceUpdate; bool needTableTagVal; bool needRequest; // whether or not request server - bool isStmtBind; // whether is stmt bind + bool isStmtBind; // whether is stmt bind } SInsertParseContext; typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); @@ -757,7 +757,7 @@ int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SS STagVal val = {0}; int32_t code = parseTagToken(pSql, pToken, pTagSchema, precision, &val, pMsgBuf); if (TSDB_CODE_SUCCESS == code) { - if (NULL == taosArrayPush(pTagVals, &val)) { + if (NULL == taosArrayPush(pTagVals, &val)){ code = terrno; } } @@ -775,14 +775,11 @@ static int32_t buildCreateTbReq(SVnodeModifyOpStmt* pStmt, STag* pTag, SArray* p return terrno; } return insBuildCreateTbReq(pStmt->pCreateTblReq, pStmt->targetTableName.tname, pTag, pStmt->pTableMeta->suid, - pStmt->usingTableName.tname, pTagName, pStmt->pTableMeta->tableInfo.numOfTags, - TSDB_DEFAULT_TABLE_TTL); + pStmt->usingTableName.tname, pTagName, pStmt->pTableMeta->tableInfo.numOfTags, + TSDB_DEFAULT_TABLE_TTL); } int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf, int8_t type) { - if (pToken->type == TK_NK_QUESTION) { - return buildInvalidOperationMsg(pMsgBuf, "insert into super table syntax is not supported for stmt"); - } if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER && pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL && pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT && pToken->type != TK_NK_BIN && @@ -813,7 +810,7 @@ typedef struct SRewriteTagCondCxt { static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) { SValueNode* pValue = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pValue); + int32_t code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pValue); if (NULL == pValue) { return code; } @@ -1044,7 +1041,7 @@ static int32_t storeChildTableMeta(SInsertParseContext* pCxt, SVnodeModifyOpStmt return TSDB_CODE_OUT_OF_MEMORY; } - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(&pStmt->targetTableName, tbFName); if (TSDB_CODE_SUCCESS != code) { taosMemoryFree(pBackup); @@ -1239,7 +1236,7 @@ static int32_t getTargetTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModi } static int32_t collectUseTable(const SName* pName, SHashObj* pTable) { - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pName, fullName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1385,7 +1382,7 @@ static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false, false); } - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(&pStmt->targetTableName, tbFName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1926,8 +1923,8 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod } if (code == TSDB_CODE_SUCCESS) { code = insBuildCreateTbReq(pStbRowsCxt->pCreateCtbReq, pStbRowsCxt->ctbName.tname, pStbRowsCxt->pTag, - pStbRowsCxt->pStbMeta->uid, pStbRowsCxt->stbName.tname, pStbRowsCxt->aTagNames, - getNumOfTags(pStbRowsCxt->pStbMeta), TSDB_DEFAULT_TABLE_TTL); + pStbRowsCxt->pStbMeta->uid, pStbRowsCxt->stbName.tname, pStbRowsCxt->aTagNames, + getNumOfTags(pStbRowsCxt->pStbMeta), TSDB_DEFAULT_TABLE_TTL); pStbRowsCxt->pTag = NULL; } @@ -1936,9 +1933,9 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod code = tNameExtractFullName(&pStbRowsCxt->ctbName, ctbFName); SVgroupInfo vg; SRequestConnInfo conn = {.pTrans = pCxt->pComCxt->pTransporter, - .requestId = pCxt->pComCxt->requestId, - .requestObjRefId = pCxt->pComCxt->requestRid, - .mgmtEps = pCxt->pComCxt->mgmtEpSet}; + .requestId = pCxt->pComCxt->requestId, + .requestObjRefId = pCxt->pComCxt->requestRid, + .mgmtEps = pCxt->pComCxt->mgmtEpSet}; if (TSDB_CODE_SUCCESS == code) { code = catalogGetTableHashVgroup(pCxt->pComCxt->pCatalog, &conn, &pStbRowsCxt->ctbName, &vg); } @@ -2179,8 +2176,8 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt if (code == TSDB_CODE_SUCCESS) { SStbRowsDataContext* pStbRowsCxt = rowsDataCxt.pStbRowsCxt; void* pData = pTableDataCxt; - code = taosHashPut(pStmt->pTableCxtHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), - &pData, POINTER_BYTES); + code = taosHashPut(pStmt->pTableCxtHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), &pData, + POINTER_BYTES); if (TSDB_CODE_SUCCESS != code) { break; } @@ -2252,7 +2249,7 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt if (!pStmt->stbSyntax && numOfRows > 0) { void* pData = rowsDataCxt.pTableDataCxt; code = taosHashPut(pStmt->pTableCxtHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), &pData, - POINTER_BYTES); + POINTER_BYTES); } return code; @@ -2366,7 +2363,8 @@ static int32_t constructStbRowsDataContext(SVnodeModifyOpStmt* pStmt, SStbRowsDa if (TSDB_CODE_SUCCESS == code) { // col values and bound cols info of STableDataContext is not used pStbRowsCxt->aColVals = taosArrayInit(getNumOfColumns(pStbRowsCxt->pStbMeta), sizeof(SColVal)); - if (!pStbRowsCxt->aColVals) code = terrno; + if (!pStbRowsCxt->aColVals) + code = terrno; } if (TSDB_CODE_SUCCESS == code) { code = insInitColValues(pStbRowsCxt->pStbMeta, pStbRowsCxt->aColVals); @@ -2424,6 +2422,9 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif // 1. [(tag1_name, ...)] ... // 2. VALUES ... | FILE ... static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + if (pStmt->stbSyntax && TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { + return buildInvalidOperationMsg(&pCxt->msg, "insert into super table syntax is not supported for stmt"); + } if (!pStmt->stbSyntax) { STableDataCxt* pTableCxt = NULL; int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); @@ -2510,9 +2511,9 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif } // db.? situation，ensure that the only thing following the '.' mark is '?' - char* tbNameAfterDbName = strnchr(pTbName->z, '.', pTbName->n, true); + char *tbNameAfterDbName = strnchr(pTbName->z, '.', pTbName->n, true); if ((tbNameAfterDbName != NULL) && (*(tbNameAfterDbName + 1) == '?')) { - char* tbName = NULL; + char *tbName = NULL; if (NULL == pCxt->pComCxt->pStmtCb) { return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", pTbName->z); } @@ -2527,8 +2528,7 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif if (pCxt->isStmtBind) { if (TK_NK_ID == pTbName->type || (tbNameAfterDbName != NULL && *(tbNameAfterDbName + 1) != '?')) { // In SQL statements, the table name has already been specified. - parserWarn("0x%" PRIx64 " table name is specified in sql, ignore the table name in bind param", - pCxt->pComCxt->requestId); + parserWarn("0x%" PRIx64 " table name is specified in sql, ignore the table name in bind param", pCxt->pComCxt->requestId); } } @@ -2614,7 +2614,7 @@ static void destroySubTableHashElem(void* p) { taosMemoryFree(*(STableMeta**)p); static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, SNode** pOutput) { SVnodeModifyOpStmt* pStmt = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT, (SNode**)&pStmt); + int32_t code = nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT, (SNode**)&pStmt); if (NULL == pStmt) { return code; } @@ -2729,7 +2729,7 @@ static int32_t buildTagNameFromMeta(STableMeta* pMeta, SArray** pTagName) { return terrno; } SSchema* pSchema = getTableTagSchema(pMeta); - int32_t code = 0; + int32_t code = 0; for (int32_t i = 0; i < pMeta->tableInfo.numOfTags; ++i) { if (NULL == taosArrayPush(*pTagName, pSchema[i].name)) { code = terrno; @@ -2834,7 +2834,7 @@ static int32_t resetVnodeModifOpStmt(SInsertParseContext* pCxt, SQuery* pQuery) } if (NULL == pStmt->pTableBlockHashObj) { pStmt->pTableBlockHashObj = - taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } if (NULL == pStmt->pVgroupsHashObj || NULL == pStmt->pTableBlockHashObj) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -2866,7 +2866,7 @@ static int32_t initInsertQuery(SInsertParseContext* pCxt, SCatalogReq* pCatalogR static int32_t setRefreshMeta(SQuery* pQuery) { SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; - int32_t code = 0; + int32_t code = 0; if (taosHashGetSize(pStmt->pTableNameHashObj) > 0) { taosArrayDestroy(pQuery->pTableList); @@ -3065,10 +3065,9 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal .forceUpdate = (NULL != pCatalogReq ? pCatalogReq->forceUpdate : false), .isStmtBind = pCxt->isStmtBind}; - int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery); - SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)((*pQuery)->pRoot); + int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery); if (TSDB_CODE_SUCCESS == code) { - code = parseInsertSqlImpl(&context, pStmt); + code = parseInsertSqlImpl(&context, (SVnodeModifyOpStmt*)(*pQuery)->pRoot); } if (TSDB_CODE_SUCCESS == code) { code = setNextStageInfo(&context, *pQuery, pCatalogReq); @@ -3077,27 +3076,8 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal QUERY_EXEC_STAGE_SCHEDULE == (*pQuery)->execStage) { code = setRefreshMeta(*pQuery); } - - if (pStmt->stbSyntax && TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && - code == TSDB_CODE_TSC_INVALID_OPERATION) { - context.tags.numOfBound = pStmt->pStbRowsCxt->boundColsInfo.numOfBound; - context.tags.numOfCols = pStmt->pStbRowsCxt->boundColsInfo.numOfCols; - context.tags.hasBoundCols = pStmt->pStbRowsCxt->boundColsInfo.hasBoundCols; - context.tags.pColIndex = taosMemoryMalloc(sizeof(int16_t) * context.tags.numOfBound); - if (NULL == context.tags.pColIndex) { - return terrno; - } - - (void)memcpy(context.tags.pColIndex, pStmt->pStbRowsCxt->boundColsInfo.pColIndex, - sizeof(int16_t) * pStmt->pStbRowsCxt->boundColsInfo.numOfBound); - code = setStmtInfo(&context, pStmt); - if (TSDB_CODE_SUCCESS == code) { - insDestroyBoundColInfo(&context.tags); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - insDestroyBoundColInfo(&context.tags); + // if no data to insert, set emptyMode to avoid request server if (!context.needRequest) { (*pQuery)->execMode = QUERY_EXEC_MODE_EMPTY_RESULT; diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 1355d9e09f..ee61611bf2 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -242,7 +242,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch } code = insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName, - pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); + pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); pTag = NULL; end: @@ -594,7 +594,7 @@ int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const c } code = insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName, - pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); + pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); pTag = NULL; end: @@ -886,7 +886,7 @@ _return: int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_E** fields, uint8_t timePrec) { - if (fields != NULL) { + if (fields) { *fields = taosMemoryCalloc(numOfBound, sizeof(TAOS_FIELD_E)); if (NULL == *fields) { return terrno; @@ -939,7 +939,7 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); if (pDataBlock->boundColsInfo.numOfBound <= 0) { *fieldNum = 0; - if (fields != NULL) { + if (fields) { *fields = NULL; } diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 9c2bb6be3d..d8a4e19218 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -25,7 +25,6 @@ exe: gcc $(CFLAGS) ./stmt.c -o $(ROOT)stmt $(LFLAGS) gcc $(CFLAGS) ./stmt2.c -o $(ROOT)stmt2 $(LFLAGS) gcc $(CFLAGS) ./stmt2-example.c -o $(ROOT)stmt2-example $(LFLAGS) - gcc $(CFLAGS) ./stmt2-get-fields.c -o $(ROOT)stmt2-get-fields $(LFLAGS) gcc $(CFLAGS) ./stmt2-nohole.c -o $(ROOT)stmt2-nohole $(LFLAGS) gcc $(CFLAGS) ./stmt-crash.c -o $(ROOT)stmt-crash $(LFLAGS) @@ -43,6 +42,5 @@ clean: rm $(ROOT)stmt rm $(ROOT)stmt2 rm $(ROOT)stmt2-example - rm $(ROOT)stmt2-get-fields rm $(ROOT)stmt2-nohole rm $(ROOT)stmt-crash diff --git a/tests/script/api/stmt2-get-fields.c b/tests/script/api/stmt2-get-fields.c deleted file mode 100644 index 23b91b56c9..0000000000 --- a/tests/script/api/stmt2-get-fields.c +++ /dev/null @@ -1,63 +0,0 @@ -// TAOS standard API example. The same syntax as MySQL, but only a subet -// to compile: gcc -o stmt2-get-fields stmt2-get-fields.c -ltaos - -#include -#include -#include -#include "taos.h" - -void do_query(TAOS *taos, const char *sql) { - TAOS_RES *result = taos_query(taos, sql); - int code = taos_errno(result); - if (code) { - printf("failed to query: %s, reason:%s\n", sql, taos_errstr(result)); - taos_free_result(result); - return; - } - taos_free_result(result); -} - -void do_stmt(TAOS *taos) { - do_query(taos, "drop database if exists db"); - do_query(taos, "create database db"); - do_query(taos, - "create table db.stb (ts timestamp, b binary(10)) tags(t1 " - "int, t2 binary(10))"); - - TAOS_STMT2_OPTION option = {0}; - TAOS_STMT2 *stmt = taos_stmt2_init(taos, &option); - const char *sql = "insert into db.stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)"; - - int code = taos_stmt2_prepare(stmt, sql, 0); - if (code != 0) { - printf("failed to execute taos_stmt2_prepare. error:%s\n", taos_stmt2_error(stmt)); - taos_stmt2_close(stmt); - return; - } - - int fieldNum = 0; - TAOS_FIELD_ALL *pFields = NULL; - code = taos_stmt2_get_all_fields(stmt, &fieldNum, &pFields); - if (code != 0) { - printf("failed get col,ErrCode: 0x%x, ErrMessage: %s.\n", code, taos_stmt2_error(stmt)); - } else { - printf("col nums:%d\n", fieldNum); - for (int i = 0; i < fieldNum; i++) { - printf("field[%d]: %s,type:%d\n", i, pFields[i].name, pFields[i].field_type); - } - } - - taos_stmt2_close(stmt); -} - -int main() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", "", 0); - if (!taos) { - printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); - exit(1); - } - - do_stmt(taos); - taos_close(taos); - taos_cleanup(); -} \ No newline at end of file From 3a66d2fb3bbec8c0a0bc7c3589ed61e5fd77f4ea Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 31 Oct 2024 14:51:56 +0800 Subject: [PATCH 33/34] fix:[TS-5441] error if cols not equal in write_raw_block --- source/libs/parser/src/parInsertUtil.c | 189 ++++++++++---------- utils/test/c/write_raw_block_test.c | 227 ++++++------------------- 2 files changed, 141 insertions(+), 275 deletions(-) diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 8adf32d2dd..bcb560ab5e 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -147,7 +147,7 @@ int16_t insFindCol(SToken* pColname, int16_t start, int16_t end, SSchema* pSchem } int32_t insBuildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid, const char* sname, - SArray* tagName, uint8_t tagNum, int32_t ttl) { + SArray* tagName, uint8_t tagNum, int32_t ttl) { pTbReq->type = TD_CHILD_TABLE; pTbReq->ctb.pTag = (uint8_t*)pTag; pTbReq->name = taosStrdup(tname); @@ -174,7 +174,7 @@ static void initBoundCols(int32_t ncols, int16_t* pBoundCols) { static int32_t initColValues(STableMeta* pTableMeta, SArray* pValues) { SSchema* pSchemas = getTableColumnSchema(pTableMeta); - int32_t code = 0; + int32_t code = 0; for (int32_t i = 0; i < pTableMeta->tableInfo.numOfColumns; ++i) { SColVal val = COL_VAL_NONE(pSchemas[i].colId, pSchemas[i].type); if (NULL == taosArrayPush(pValues, &val)) { @@ -886,19 +886,77 @@ static bool findFileds(SSchema* pSchema, TAOS_FIELD* fields, int numFields) { return false; } +int32_t checkSchema(SSchema* pColSchema, int8_t* fields, char* errstr, int32_t errstrLen) { + if (*fields != pColSchema->type) { + if (errstr != NULL) + snprintf(errstr, errstrLen, "column type not equal, name:%s, schema type:%s, data type:%s", pColSchema->name, + tDataTypes[pColSchema->type].name, tDataTypes[*fields].name); + return TSDB_CODE_INVALID_PARA; + } + if (IS_VAR_DATA_TYPE(pColSchema->type) && *(int32_t*)(fields + sizeof(int8_t)) > pColSchema->bytes) { + if (errstr != NULL) + snprintf(errstr, errstrLen, + "column var data bytes error, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d", + pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name, + *(int32_t*)(fields + sizeof(int8_t))); + return TSDB_CODE_INVALID_PARA; + } + + if (!IS_VAR_DATA_TYPE(pColSchema->type) && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { + if (errstr != NULL) + snprintf(errstr, errstrLen, + "column normal data bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d", + pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name, + *(int32_t*)(fields + sizeof(int8_t))); + return TSDB_CODE_INVALID_PARA; + } + return 0; +} + +#define PRCESS_DATA(i, j) \ + ret = checkSchema(pColSchema, fields, errstr, errstrLen); \ + if (ret != 0) { \ + goto end; \ + } \ + \ + if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { \ + hasTs = true; \ + } \ + \ + int8_t* offset = pStart; \ + if (IS_VAR_DATA_TYPE(pColSchema->type)) { \ + pStart += numOfRows * sizeof(int32_t); \ + } else { \ + pStart += BitmapLen(numOfRows); \ + } \ + char* pData = pStart; \ + \ + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j); \ + ret = tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData); \ + if (ret != 0) { \ + goto end; \ + } \ + fields += sizeof(int8_t) + sizeof(int32_t); \ + if (needChangeLength && version == BLOCK_VERSION_1) { \ + pStart += htonl(colLength[i]); \ + } else { \ + pStart += colLength[i]; \ + } \ + boundInfo->pColIndex[j] = -1; + int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* tFields, int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw) { int ret = 0; - if(data == NULL) { + if (data == NULL) { uError("rawBlockBindData, data is NULL"); return TSDB_CODE_APP_ERROR; } void* tmp = taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); - SVCreateTbReq *pCreateReqTmp = NULL; - if (tmp == NULL && pCreateTb != NULL){ + SVCreateTbReq* pCreateReqTmp = NULL; + if (tmp == NULL && pCreateTb != NULL) { ret = cloneSVreateTbReq(pCreateTb, &pCreateReqTmp); - if (ret != TSDB_CODE_SUCCESS){ + if (ret != TSDB_CODE_SUCCESS) { uError("cloneSVreateTbReq error"); goto end; } @@ -906,7 +964,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate STableDataCxt* pTableCxt = NULL; ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateReqTmp, &pTableCxt, true, false); + sizeof(pTableMeta->uid), pTableMeta, &pCreateReqTmp, &pTableCxt, true, false); if (pCreateReqTmp != NULL) { tdDestroySVCreateTbReq(pCreateReqTmp); taosMemoryFree(pCreateReqTmp); @@ -963,121 +1021,48 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate ret = TSDB_CODE_INVALID_PARA; goto end; } -// if (tFields != NULL && numFields > boundInfo->numOfBound) { -// if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d bigger than num of bound cols:%d", numFields, boundInfo->numOfBound); -// ret = TSDB_CODE_INVALID_PARA; -// goto end; -// } - if (tFields == NULL && numOfCols != boundInfo->numOfBound) { - if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d not equal to num of bound cols:%d", numOfCols, boundInfo->numOfBound); - ret = TSDB_CODE_INVALID_PARA; - goto end; - } + bool hasTs = false; if (tFields == NULL) { - for (int j = 0; j < boundInfo->numOfBound; j++) { - SSchema* pColSchema = &pSchema[j]; - SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j); - if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { - if (errstr != NULL) - snprintf(errstr, errstrLen, - "column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d", - pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name, - *(int32_t*)(fields + sizeof(int8_t))); - ret = TSDB_CODE_INVALID_PARA; - goto end; - } - - int8_t* offset = pStart; - if (IS_VAR_DATA_TYPE(pColSchema->type)) { - pStart += numOfRows * sizeof(int32_t); - } else { - pStart += BitmapLen(numOfRows); - } - char* pData = pStart; - ret = tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData); - if (ret != 0) { - goto end; - } - fields += sizeof(int8_t) + sizeof(int32_t); - if (needChangeLength && version == BLOCK_VERSION_1) { - pStart += htonl(colLength[j]); - } else { - pStart += colLength[j]; - } + int32_t len = TMIN(numOfCols, boundInfo->numOfBound); + for (int j = 0; j < len; j++) { + SSchema* pColSchema = &pSchema[j]; + PRCESS_DATA(j, j) } } else { - bool hasTs = false; for (int i = 0; i < numFields; i++) { for (int j = 0; j < boundInfo->numOfBound; j++) { SSchema* pColSchema = &pSchema[j]; - char* fieldName = NULL; + char* fieldName = NULL; if (raw) { fieldName = ((SSchemaWrapper*)tFields)->pSchema[i].name; } else { fieldName = ((TAOS_FIELD*)tFields)[i].name; } if (strcmp(pColSchema->name, fieldName) == 0) { - if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { - if (errstr != NULL) - snprintf(errstr, errstrLen, - "column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d", - pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name, - *(int32_t*)(fields + sizeof(int8_t))); - ret = TSDB_CODE_INVALID_PARA; - goto end; - } - - if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - hasTs = true; - } - - int8_t* offset = pStart; - if (IS_VAR_DATA_TYPE(pColSchema->type)) { - pStart += numOfRows * sizeof(int32_t); - } else { - pStart += BitmapLen(numOfRows); -// for(int k = 0; k < numOfRows; k++) { -// if(!colDataIsNull_f(offset, k) && pColSchema->type == TSDB_DATA_TYPE_INT){ -// printf("colName:%s,val:%d", fieldName, *(int32_t*)(pStart + k * sizeof(int32_t))); -// } -// } - } - char* pData = pStart; - - SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j); - ret = tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData); - if (ret != 0) { - goto end; - } - fields += sizeof(int8_t) + sizeof(int32_t); - if (needChangeLength && version == BLOCK_VERSION_1) { - pStart += htonl(colLength[i]); - } else { - pStart += colLength[i]; - } - boundInfo->pColIndex[j] = -1; + PRCESS_DATA(i, j) break; } } } + } - if (!hasTs) { - if (errstr != NULL) snprintf(errstr, errstrLen, "timestamp column(primary key) not found in raw data"); - ret = TSDB_CODE_INVALID_PARA; - goto end; - } + if (!hasTs) { + if (errstr != NULL) snprintf(errstr, errstrLen, "timestamp column(primary key) not found in raw data"); + ret = TSDB_CODE_INVALID_PARA; + goto end; + } - for (int c = 0; c < boundInfo->numOfBound; ++c) { - if (boundInfo->pColIndex[c] != -1) { - SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c); - ret = tColDataAddValueByDataBlock(pCol, 0, 0, numOfRows, NULL, NULL); - if (ret != 0) { - goto end; - } - } else { - boundInfo->pColIndex[c] = c; // restore for next block + // process NULL data + for (int c = 0; c < boundInfo->numOfBound; ++c) { + if (boundInfo->pColIndex[c] != -1) { + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c); + ret = tColDataAddValueByDataBlock(pCol, 0, 0, numOfRows, NULL, NULL); + if (ret != 0) { + goto end; } + } else { + boundInfo->pColIndex[c] = c; // restore for next block } } diff --git a/utils/test/c/write_raw_block_test.c b/utils/test/c/write_raw_block_test.c index 8ed997bc92..ae4a606e6e 100644 --- a/utils/test/c/write_raw_block_test.c +++ b/utils/test/c/write_raw_block_test.c @@ -19,196 +19,77 @@ #include "taos.h" #include "types.h" -int buildStable(TAOS* pConn) { - TAOS_RES* pRes = taos_query(pConn, - "CREATE STABLE `meters` (`ts` TIMESTAMP, `current` INT, `voltage` INT, `phase` FLOAT) TAGS " - "(`groupid` INT, `location` VARCHAR(16))"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table meters, reason:%s\n", taos_errstr(pRes)); - return -1; - } +TAOS* pConn = NULL; +void action(char* sql) { + TAOS_RES* pRes = taos_query(pConn, sql); + ASSERT(taos_errno(pRes) == 0); taos_free_result(pRes); - - pRes = taos_query(pConn, "create table d0 using meters tags(1, 'San Francisco')"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table d0, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into d0 (ts, current) values (now, 120)"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into table d0, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table d1 using meters tags(2, 'San Francisco')"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table d1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table d2 using meters tags(3, 'San Francisco')"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table d2, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table ntba(ts timestamp, addr binary(32))"); - if (taos_errno(pRes) != 0) { - printf("failed to create ntba, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table ntbb(ts timestamp, addr binary(8))"); - if (taos_errno(pRes) != 0) { - printf("failed to create ntbb, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ntba values(now,'123456789abcdefg123456789')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert table ntba, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ntba values(now + 1s,'hello')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert table ntba, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - return 0; } -int32_t init_env() { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - if (pConn == NULL) { - return -1; - } - int32_t ret = -1; - - TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_raw"); - if (taos_errno(pRes) != 0) { - printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); - goto END; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create database if not exists db_raw vgroups 2"); - if (taos_errno(pRes) != 0) { - printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); - goto END; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use db_raw"); - if (taos_errno(pRes) != 0) { - printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); - goto END; - } - taos_free_result(pRes); - - buildStable(pConn); - - pRes = taos_query(pConn, "select * from d0"); - if (taos_errno(pRes) != 0) { - printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); - goto END; - } - void *data = NULL; +int32_t test_write_raw_block(char* query, char* dst) { + TAOS_RES* pRes = taos_query(pConn, query); + ASSERT(taos_errno(pRes) == 0); + void* data = NULL; int32_t numOfRows = 0; int error_code = taos_fetch_raw_block(pRes, &numOfRows, &data); - if(error_code !=0 ){ - printf("error fetch raw block, reason:%s\n", taos_errstr(pRes)); - goto END; - } - - taos_write_raw_block(pConn, numOfRows, data, "d1"); + ASSERT(error_code == 0); + error_code = taos_write_raw_block(pConn, numOfRows, data, dst); taos_free_result(pRes); + return error_code; +} - pRes = taos_query(pConn, "select ts,phase from d0"); - if (taos_errno(pRes) != 0) { - printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); - goto END; - } - error_code = taos_fetch_raw_block(pRes, &numOfRows, &data); - if(error_code !=0 ){ - printf("error fetch raw block, reason:%s\n", taos_errstr(pRes)); - goto END; - } +int32_t test_write_raw_block_with_fields(char* query, char* dst) { + TAOS_RES* pRes = taos_query(pConn, query); + ASSERT(taos_errno(pRes) == 0); + void* data = NULL; + int32_t numOfRows = 0; + int error_code = taos_fetch_raw_block(pRes, &numOfRows, &data); + ASSERT(error_code == 0); int numFields = taos_num_fields(pRes); - TAOS_FIELD *fields = taos_fetch_fields(pRes); - taos_write_raw_block_with_fields(pConn, numOfRows, data, "d2", fields, numFields); + TAOS_FIELD* fields = taos_fetch_fields(pRes); + error_code = taos_write_raw_block_with_fields(pConn, numOfRows, data, dst, fields, numFields); taos_free_result(pRes); + return error_code; +} - // check error msg - pRes = taos_query(pConn, "select * from ntba"); - if (taos_errno(pRes) != 0) { - printf("error select * from ntba, reason:%s\n", taos_errstr(pRes)); - goto END; - } +void init_env() { + pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT(pConn); - data = NULL; - numOfRows = 0; - error_code = taos_fetch_raw_block(pRes, &numOfRows, &data); - if(error_code !=0 ){ - printf("error fetch select * from ntba, reason:%s\n", taos_errstr(pRes)); - goto END; - } - error_code = taos_write_raw_block(pConn, numOfRows, data, "ntbb"); - if(error_code == 0) { - printf(" taos_write_raw_block to ntbb expect failed , but success!\n"); - goto END; - } + action("drop database if exists db_raw"); + action("create database if not exists db_raw vgroups 2"); + action("use db_raw"); - // pass NULL return last error code describe - const char* err = tmq_err2str(error_code); - printf("write_raw_block return code =0x%x err=%s\n", error_code, err); - if(strcmp(err, "success") == 0) { - printf("expect failed , but error string is success! err=%s\n", err); - goto END; - } + action( + "CREATE STABLE `meters` (`ts` TIMESTAMP, `current` INT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, " + "`location` VARCHAR(16))"); + action("create table d0 using meters tags(1, 'San Francisco')"); + action("create table d1 using meters tags(2, 'San Francisco')"); + action("create table d2 using meters tags(3, 'San Francisco')"); + action("insert into d0 (ts, current) values (now, 120)"); - // no exist table - error_code = taos_write_raw_block(pConn, numOfRows, data, "no-exist-table"); - if(error_code == 0) { - printf(" taos_write_raw_block to no-exist-table expect failed , but success!\n"); - goto END; - } + action("create table ntba(ts timestamp, addr binary(32))"); + action("create table ntbb(ts timestamp, addr binary(8))"); + action("create table ntbc(ts timestamp, addr binary(8), c2 int)"); - err = tmq_err2str(error_code); - printf("write_raw_block no exist table return code =0x%x err=%s\n", error_code, err); - if(strcmp(err, "success") == 0) { - printf("expect failed write no exist table, but error string is success! err=%s\n", err); - goto END; - } - - // success - ret = 0; - -END: - // free - if(pRes) taos_free_result(pRes); - if(pConn) taos_close(pConn); - return ret; + action("insert into ntba values(now,'123456789abcdefg123456789')"); + action("insert into ntbb values(now + 1s,'hello')"); + action("insert into ntbc values(now + 13s, 'sdf', 123)"); } int main(int argc, char* argv[]) { - printf("test write_raw_block...\n"); - int ret = init_env(); - if (ret < 0) { - printf("test write_raw_block failed.\n"); - return ret; - } - printf("test write_raw_block ok.\n"); + printf("test write_raw_block start.\n"); + init_env(); + ASSERT(test_write_raw_block("select * from d0", "d1") == 0); // test schema same + ASSERT(test_write_raw_block("select * from ntbb", "ntba") == 0); // test schema compatible + ASSERT(test_write_raw_block("select * from ntbb", "ntbc") == 0); // test schema small + ASSERT(test_write_raw_block("select * from ntbc", "ntbb") == 0); // test schema bigger + ASSERT(test_write_raw_block("select * from ntba", "ntbb") != 0); // test schema mismatch + ASSERT(test_write_raw_block("select * from ntba", "no-exist-table") != 0); // test no exist table + ASSERT(test_write_raw_block("select addr from ntba", "ntbb") != 0); // test without ts + ASSERT(test_write_raw_block_with_fields("select ts,phase from d0", "d2") == 0); // test with fields + + printf("test write_raw_block end.\n"); return 0; } \ No newline at end of file From 2879fafc1c6e9a27e268e630336949f3d54d1183 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 31 Oct 2024 14:54:01 +0800 Subject: [PATCH 34/34] fix: typo issue --- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 1ae7dfeff7..2cef541cdb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1378,7 +1378,7 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 } else { if (!iColHash) { iColHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)); - if (ctxArray == NULL) { + if (iColHash == NULL) { tsdbRowClose(&iter); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit); }