From 22c44444e9aa93089f85451c244bc80a3795ec84 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 29 Sep 2024 19:59:46 +0800 Subject: [PATCH 01/61] fix double send --- source/libs/transport/src/thttp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index a4cfa69459..7d7868f3cd 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -677,7 +677,7 @@ static void httpHandleReq(SHttpMsg* msg) { tError("http-report failed to connect to http-server,dst:%s:%d, chanId:%" PRId64 ", seq:%" PRId64 ", reson:%s", cli->addr, cli->port, chanId, cli->seq, uv_strerror(ret)); httpFailFastMayUpdate(http->connStatusTable, cli->addr, cli->port, 0); - destroyHttpClient(cli); + uv_close((uv_handle_t*)&cli->tcp, httpDestroyClientCb); } TAOS_UNUSED(taosReleaseRef(httpRefMgt, chanId)); return; From 6e43b1a69491256aafd268c1adfccc98fd60b500 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 29 Sep 2024 20:01:38 +0800 Subject: [PATCH 02/61] formate qid --- source/client/src/clientRawBlockWrite.c | 42 +++++++------- source/dnode/vnode/src/tqCommon/tqCommon.c | 9 ++- source/libs/catalog/src/ctgAsync.c | 65 +++++++++++----------- source/libs/catalog/src/ctgRemote.c | 36 +++++++----- source/libs/catalog/src/ctgUtil.c | 19 ++++--- source/libs/planner/src/planner.c | 6 +- source/libs/scheduler/src/schJob.c | 24 ++++---- source/libs/scheduler/src/schRemote.c | 4 +- source/libs/scheduler/src/schTask.c | 8 +-- source/libs/stream/src/streamCheckStatus.c | 18 +++--- 10 files changed, 123 insertions(+), 108 deletions(-) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index d0ea7055de..8ce4685716 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -23,12 +23,12 @@ #include "tglobal.h" #include "tmsgtype.h" -#define RAW_NULL_CHECK(c) \ - do { \ - if (c == NULL) { \ - code = terrno; \ - goto end; \ - } \ +#define RAW_NULL_CHECK(c) \ + do { \ + if (c == NULL) { \ + code = terrno; \ + goto end; \ + } \ } while (0) #define RAW_FALSE_CHECK(c) \ @@ -47,7 +47,7 @@ } \ } while (0) -#define LOG_ID_TAG "connId:0x%" PRIx64 ",qid:0x%" PRIx64 +#define LOG_ID_TAG "connId:0x%" PRIx64 ",QID:0x%" PRIx64 #define LOG_ID_VALUE *(int64_t*)taos, pRequest->requestId #define TMQ_META_VERSION "1.0" @@ -1188,7 +1188,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { pCreateReq->ctb.suid = pTableMeta->uid; SArray* pTagVals = NULL; - code = tTagToValArray((STag *)pCreateReq->ctb.pTag, &pTagVals); + code = tTagToValArray((STag*)pCreateReq->ctb.pTag, &pTagVals); if (code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pTableMeta); goto end; @@ -1206,18 +1206,19 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { if (strcmp(tag->name, tName) == 0 && tag->type != TSDB_DATA_TYPE_JSON) { STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, i); if (pTagVal) { - if (pTagVal->cid != tag->colId){ + if (pTagVal->cid != tag->colId) { pTagVal->cid = tag->colId; rebuildTag = true; } } else { - uError("create tb invalid data %s, size:%d index:%d cid:%d", pCreateReq->name, (int)taosArrayGetSize(pTagVals), i, tag->colId); + uError("create tb invalid data %s, size:%d index:%d cid:%d", pCreateReq->name, + (int)taosArrayGetSize(pTagVals), i, tag->colId); } } } } taosMemoryFreeClear(pTableMeta); - if (rebuildTag){ + if (rebuildTag) { STag* ppTag = NULL; code = tTagNew(pTagVals, 1, false, &ppTag); taosArrayDestroy(pTagVals); @@ -1815,7 +1816,7 @@ end: static int32_t buildCreateTbMap(STaosxRsp* rsp, SHashObj* pHashObj) { // find schema data info - int32_t code = 0; + int32_t code = 0; SVCreateTbReq pCreateReq = {0}; SDecoder decoderTmp = {0}; @@ -1826,15 +1827,16 @@ static int32_t buildCreateTbMap(STaosxRsp* rsp, SHashObj* pHashObj) { RAW_NULL_CHECK(lenTmp); tDecoderInit(&decoderTmp, *dataTmp, *lenTmp); - RAW_RETURN_CHECK (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq)); + RAW_RETURN_CHECK(tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq)); if (pCreateReq.type != TSDB_CHILD_TABLE) { code = TSDB_CODE_INVALID_MSG; goto end; } - if (taosHashGet(pHashObj, pCreateReq.name, strlen(pCreateReq.name)) == NULL){ - RAW_RETURN_CHECK(taosHashPut(pHashObj, pCreateReq.name, strlen(pCreateReq.name), &pCreateReq, sizeof(SVCreateTbReq))); - } else{ + if (taosHashGet(pHashObj, pCreateReq.name, strlen(pCreateReq.name)) == NULL) { + RAW_RETURN_CHECK( + taosHashPut(pHashObj, pCreateReq.name, strlen(pCreateReq.name), &pCreateReq, sizeof(SVCreateTbReq))); + } else { tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); pCreateReq = (SVCreateTbReq){0}; } @@ -1927,7 +1929,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) // find schema data info SVCreateTbReq* pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName)); - SVgroupInfo vg = {0}; + SVgroupInfo vg = {0}; RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg)); if (pCreateReqDst) { // change stable name to get meta (void)strcpy(pName.tname, pCreateReqDst->ctb.stbName); @@ -1957,10 +1959,10 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) fields[i].bytes = pSW->pSchema[i].bytes; tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name)); } - void* rawData = getRawDataFromRes(pRetrieve); - char err[ERR_MSG_LEN] = {0}; + void* rawData = getRawDataFromRes(pRetrieve); + char err[ERR_MSG_LEN] = {0}; SVCreateTbReq* pCreateReqTmp = NULL; - if (pCreateReqDst){ + if (pCreateReqDst) { RAW_RETURN_CHECK(cloneSVreateTbReq(pCreateReqDst, &pCreateReqTmp)); } code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqTmp, fields, pSW->nCols, true, err, ERR_MSG_LEN); diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 30be253b65..6b7e857120 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -84,7 +84,6 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) { code = qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId); if (code) { - return code; } } @@ -363,7 +362,7 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { } pRspHead->vgId = htonl(req.upstreamNodeId); - if(pRspHead->vgId == 0) { + if (pRspHead->vgId == 0) { tqError("vgId:%d invalid dispatch msg from upstream to task:0x%x", pMeta->vgId, req.taskId); return TSDB_CODE_INVALID_MSG; } @@ -460,7 +459,7 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { if (code != TSDB_CODE_SUCCESS) { // return error not send rsp manually tqError("s-task:0x%x vgId:%d failed to process retrieve request from 0x%x, code:%s", req.dstTaskId, req.dstNodeId, req.srcTaskId, tstrerror(code)); - } else { // send rsp manually only on success. + } else { // send rsp manually only on success. SRpcMsg rsp = {.info = pMsg->info, .code = 0}; streamTaskSendRetrieveRsp(&req, &rsp); } @@ -515,7 +514,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe } tDecoderClear(&decoder); - tqDebug("tq task:0x%x (vgId:%d) recv check rsp(qid:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", rsp.upstreamTaskId, + tqDebug("tq task:0x%x (vgId:%d) recv check rsp(QID:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); if (!isLeader) { @@ -1272,7 +1271,7 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { streamMutexLock(&pTask->lock); if (pTask->chkInfo.checkpointId < req.checkpointId) { - tqFatal("s-task:%s vgId:%d invalid consensus-checkpointId:%" PRId64 ", greater than existed checkpointId:%"PRId64, + tqFatal("s-task:%s vgId:%d invalid consensus-checkpointId:%" PRId64 ", greater than existed checkpointId:%" PRId64, pTask->id.idStr, vgId, req.checkpointId, pTask->chkInfo.checkpointId); streamMutexUnlock(&pTask->lock); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 46f4f86484..525573ee01 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -72,7 +72,7 @@ int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -94,7 +94,7 @@ int32_t ctgInitGetTbMetasTask(SCtgJob* pJob, int32_t taskIdx, void* param) { ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes)); if (NULL == ctx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbMetaNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbMetaNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -105,7 +105,7 @@ int32_t ctgInitGetTbMetasTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbMetaNum); return TSDB_CODE_SUCCESS; @@ -133,7 +133,7 @@ int32_t ctgInitGetDbVgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; @@ -161,7 +161,7 @@ int32_t ctgInitGetDbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; @@ -189,7 +189,7 @@ int32_t ctgInitGetDbInfoTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; @@ -223,7 +223,7 @@ int32_t ctgInitGetTbHashTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tableName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tableName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -245,7 +245,7 @@ int32_t ctgInitGetTbHashsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes)); if (NULL == ctx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbHashNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbHashNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -256,7 +256,7 @@ int32_t ctgInitGetTbHashsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbHashNum); return TSDB_CODE_SUCCESS; @@ -275,7 +275,7 @@ int32_t ctgInitGetQnodeTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } @@ -293,7 +293,7 @@ int32_t ctgInitGetDnodeTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } @@ -320,7 +320,7 @@ int32_t ctgInitGetIndexTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, indexFName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, indexFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); return TSDB_CODE_SUCCESS; @@ -348,7 +348,7 @@ int32_t ctgInitGetUdfTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, udfName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, udfName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); return TSDB_CODE_SUCCESS; @@ -376,7 +376,7 @@ int32_t ctgInitGetUserTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, user:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, user:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), user->user); return TSDB_CODE_SUCCESS; @@ -394,7 +394,7 @@ int32_t ctgInitGetSvrVerTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } @@ -426,7 +426,7 @@ int32_t ctgInitGetTbIndexTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -459,7 +459,7 @@ int32_t ctgInitGetTbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -492,7 +492,7 @@ int32_t ctgInitGetTbTagTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; @@ -514,7 +514,7 @@ int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->viewNum, sizeof(SMetaRes)); if (NULL == ctx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->viewNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->viewNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -525,7 +525,7 @@ int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(terrno); } - qDebug("qid:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, viewNum:%d", pJob->queryId, taskIdx, + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, viewNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->viewNum); return TSDB_CODE_SUCCESS; @@ -546,7 +546,7 @@ int32_t ctgInitGetTbTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tbTsmaNum, sizeof(SMetaRes)); if (NULL == pTaskCtx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbTsmaNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbTsmaNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -574,7 +574,7 @@ int32_t ctgInitGetTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tsmaNum, sizeof(SMetaRes)); if (NULL == pTaskCtx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tsmaNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tsmaNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -603,7 +603,7 @@ static int32_t ctgInitGetTbNamesTask(SCtgJob* pJob, int32_t taskId, void* param) pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tbNameNum, sizeof(SMetaRes)); if (NULL == pTaskCtx->pResList) { - qError("qid:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbNameNum, + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbNameNum, (int32_t)sizeof(SMetaRes)); ctgFreeTask(&task, true); CTG_ERR_RET(terrno); @@ -1048,7 +1048,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const } double el = (taosGetTimestampUs() - st) / 1000.0; - qDebug("qid:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d, elapsed time:%.2f ms", + qDebug("QID:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d, elapsed time:%.2f ms", pJob->queryId, pJob->refId, taskNum, pReq->forceUpdate, el); return TSDB_CODE_SUCCESS; @@ -1450,16 +1450,17 @@ _return: int32_t ctgCallUserCb(void* param) { SCtgJob* pJob = (SCtgJob*)param; - qDebug("qid:0x%" PRIx64 " ctg start to call user cb with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); + qDebug("QID:0x%" PRIx64 " ctg start to call user cb with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); (*pJob->userFp)(&pJob->jobRes, pJob->userParam, pJob->jobResCode); - qDebug("qid:0x%" PRIx64 " ctg end to call user cb", pJob->queryId); + qDebug("QID:0x%" PRIx64 " ctg end to call user cb", pJob->queryId); int64_t refId = pJob->refId; int32_t code = taosRemoveRef(gCtgMgmt.jobPool, refId); if (code) { - qError("qid:0x%" PRIx64 " remove ctg job %" PRId64 " from jobPool failed, error:%s", pJob->queryId, refId, tstrerror(code)); + qError("QID:0x%" PRIx64 " remove ctg job %" PRId64 " from jobPool failed, error:%s", pJob->queryId, refId, + tstrerror(code)); } return TSDB_CODE_SUCCESS; @@ -1469,7 +1470,7 @@ void ctgUpdateJobErrCode(SCtgJob* pJob, int32_t errCode) { if (!NEED_CLIENT_REFRESH_VG_ERROR(errCode) || errCode == TSDB_CODE_SUCCESS) return; atomic_store_32(&pJob->jobResCode, errCode); - qDebug("qid:0x%" PRIx64 " ctg job errCode updated to %s", pJob->queryId, tstrerror(errCode)); + qDebug("QID:0x%" PRIx64 " ctg job errCode updated to %s", pJob->queryId, tstrerror(errCode)); return; } @@ -1481,7 +1482,7 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { return TSDB_CODE_SUCCESS; } - qDebug("qid:0x%" PRIx64 " task %d end with res %s", pJob->queryId, pTask->taskId, tstrerror(rspCode)); + qDebug("QID:0x%" PRIx64 " task %d end with res %s", pJob->queryId, pTask->taskId, tstrerror(rspCode)); pTask->code = rspCode; pTask->status = CTG_TASK_DONE; @@ -1490,7 +1491,7 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1); if (taskDone < taosArrayGetSize(pJob->pTasks)) { - qDebug("qid:0x%" PRIx64 " task done: %d, total: %d", pJob->queryId, taskDone, + qDebug("QID:0x%" PRIx64 " task done: %d, total: %d", pJob->queryId, taskDone, (int32_t)taosArrayGetSize(pJob->pTasks)); ctgUpdateJobErrCode(pJob, rspCode); @@ -4347,7 +4348,7 @@ int32_t ctgLaunchJob(SCtgJob* pJob) { CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("qid:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); + qDebug("QID:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); pTask = taosArrayGet(pJob->pTasks, i); @@ -4360,7 +4361,7 @@ int32_t ctgLaunchJob(SCtgJob* pJob) { } if (taskNum <= 0) { - qDebug("qid:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); + qDebug("QID:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); CTG_ERR_RET(taosAsyncExec(ctgCallUserCb, pJob, NULL)); #if CTG_BATCH_FETCH diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index d6e941c819..ed9dc81dd7 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -47,7 +47,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu msgNum = 0; } - ctgDebug("qid:0x%" PRIx64 " ctg got batch %d rsp %s", pJob->queryId, cbParam->batchId, + ctgDebug("QID:0x%" PRIx64 " ctg got batch %d rsp %s", pJob->queryId, cbParam->batchId, TMSG_INFO(cbParam->reqType + 1)); SHashObj* pBatchs = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -114,7 +114,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu pMsgCtx->pBatchs = pBatchs; - ctgDebug("qid:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId, + ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId, pRsp->msgIdx, TMSG_INFO(taskMsg.msgType + 1), pBatchs); (void)(*gCtgAsyncFps[pTask->type].handleRspFp)( @@ -454,7 +454,7 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) { CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("qid:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, + qDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1)); #if CTG_BATCH_FETCH @@ -702,7 +702,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT if (TDMT_VND_TABLE_CFG == msgType) { SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; pName = ctx->pName; - } else if (TDMT_VND_TABLE_META == msgType || TDMT_VND_TABLE_NAME == msgType) { + } else if (TDMT_VND_TABLE_META == msgType || TDMT_VND_TABLE_NAME == msgType) { if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); @@ -808,7 +808,7 @@ int32_t ctgLaunchBatchs(SCatalog* pCtg, SCtgJob* pJob, SHashObj* pBatchs) { SCtgBatch* pBatch = (SCtgBatch*)p; int32_t msgSize = 0; - ctgDebug("qid:0x%" PRIx64 " ctg start to launch batch %d", pJob->queryId, pBatch->batchId); + ctgDebug("QID:0x%" PRIx64 " ctg start to launch batch %d", pJob->queryId, pBatch->batchId); CTG_ERR_JRET(ctgBuildBatchReqMsg(pBatch, *vgId, &msg, &msgSize)); code = ctgAsyncSendMsg(pCtg, &pBatch->conn, pJob, pBatch->pTaskIds, pBatch->batchId, pBatch->pMsgIdxs, @@ -1124,10 +1124,11 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n int32_t code = tNameExtractFullName(name, tbFName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), name->type, name->dbname, name->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), name->type, + name->dbname, name->tname); CTG_ERR_RET(code); } - + code = queryBuildMsg[TMSG_INDEX(reqType)]((void*)tbFName, &msg, 0, &msgLen, mallocFp); if (code) { ctgError("Build get index msg failed, code:%s, tbFName:%s", tstrerror(code), tbFName); @@ -1450,7 +1451,8 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t code = tNameExtractFullName(pTableName, tbFName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, pTableName->dbname, pTableName->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, + pTableName->dbname, pTableName->tname); CTG_ERR_RET(code); } @@ -1523,7 +1525,8 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t code = tNameExtractFullName(pTableName, tbFName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, pTableName->dbname, pTableName->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, + pTableName->dbname, pTableName->tname); CTG_ERR_RET(code); } @@ -1632,10 +1635,11 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* int32_t reqType = TDMT_MND_VIEW_META; SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pName, fullName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pName->type, pName->dbname, pName->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pName->type, + pName->dbname, pName->tname); CTG_ERR_RET(code); } @@ -1693,10 +1697,11 @@ int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa int32_t msgLen = 0; SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(name, tbFName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), name->type, name->dbname, name->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), name->type, + name->dbname, name->tname); CTG_ERR_RET(code); } @@ -1757,10 +1762,11 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pTbName, tbFName); if (code) { - ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTbName->type, pTbName->dbname, pTbName->tname); + ctgError("tNameExtractFullName failed, code:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTbName->type, + pTbName->dbname, pTbName->tname); CTG_ERR_RET(code); } - + SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 96cd783d2f..86a38017bd 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -452,7 +452,8 @@ void ctgClearHandleMeta(SCatalog* pCtg, int64_t* pClearedSize, int64_t* pCleardN code = taosHashRemove(dbCache->tbCache, key, len); if (code) { - qError("taosHashRemove table cache failed, key:%s, len:%d, error:%s", (char*)key, (int32_t)len, tstrerror(code)); + qError("taosHashRemove table cache failed, key:%s, len:%d, error:%s", (char*)key, (int32_t)len, + tstrerror(code)); } cacheSize = @@ -1096,7 +1097,7 @@ void ctgFreeJob(void* job) { taosMemoryFree(job); - qDebug("qid:0x%" PRIx64 ", ctg job 0x%" PRIx64 " freed", qid, rid); + qDebug("QID:0x%" PRIx64 ", ctg job 0x%" PRIx64 " freed", qid, rid); } int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target) { @@ -1241,10 +1242,11 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* d char tbFullName[TSDB_TABLE_FNAME_LEN]; code = tNameExtractFullName(pTableName, tbFullName); if (code) { - ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, pTableName->dbname, pTableName->tname); + ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), pTableName->type, + pTableName->dbname, pTableName->tname); CTG_ERR_RET(code); } - + uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); @@ -1704,7 +1706,8 @@ int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) { } int32_t ctgUpdateSendTargetInfo(SMsgSendInfo* pMsgSendInfo, int32_t msgType, char* dbFName, int32_t vgId) { - if (msgType == TDMT_VND_TABLE_META || msgType == TDMT_VND_TABLE_CFG || msgType == TDMT_VND_BATCH_META || msgType == TDMT_VND_TABLE_NAME) { + if (msgType == TDMT_VND_TABLE_META || msgType == TDMT_VND_TABLE_CFG || msgType == TDMT_VND_BATCH_META || + msgType == TDMT_VND_TABLE_NAME) { pMsgSendInfo->target.type = TARGET_TYPE_VNODE; pMsgSendInfo->target.vgId = vgId; pMsgSendInfo->target.dbFName = taosStrdup(dbFName); @@ -2010,7 +2013,8 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { char dbFName[TSDB_DB_FNAME_LEN]; code = tNameExtractFullName(&req->pRawReq->tbName, tbFName); if (code) { - ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), req->pRawReq->tbName.type, req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); + ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), + req->pRawReq->tbName.type, req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); CTG_ERR_RET(code); } @@ -2201,7 +2205,8 @@ int32_t ctgChkSetViewAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) } else { code = tNameExtractFullName(&req->pRawReq->tbName, viewFName); if (code) { - ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), req->pRawReq->tbName.type, req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); + ctgError("tNameExtractFullName failed, error:%s, type:%d, dbName:%s, tname:%s", tstrerror(code), + req->pRawReq->tbName.type, req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); CTG_ERR_RET(code); } } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 43159bce20..c3aa95f5b7 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -37,7 +37,7 @@ static int32_t dumpQueryPlan(SQueryPlan* pPlan) { char* pStr = NULL; code = nodesNodeToString((SNode*)pPlan, false, &pStr, NULL); if (TSDB_CODE_SUCCESS == code) { - planDebugL("qid:0x%" PRIx64 " Query Plan, JsonPlan: %s", pPlan->queryId, pStr); + planDebugL("QID:0x%" PRIx64 " Query Plan, JsonPlan: %s", pPlan->queryId, pStr); taosMemoryFree(pStr); } return code; @@ -123,7 +123,7 @@ int32_t qContinuePlanPostQuery(void* pPostPlan) { } int32_t qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstreamSourceNode* pSource) { - planDebug("qid:0x%" PRIx64 " set subplan execution node, groupId:%d", subplan->id.queryId, groupId); + planDebug("QID:0x%" PRIx64 " set subplan execution node, groupId:%d", subplan->id.queryId, groupId); return setSubplanExecutionNode(subplan->pNode, groupId, pSource); } @@ -143,7 +143,7 @@ static void clearSubplanExecutionNode(SPhysiNode* pNode) { } void qClearSubplanExecutionNode(SSubplan* pSubplan) { - planDebug("qid:0x%" PRIx64 " clear subplan execution node, groupId:%d", pSubplan->id.queryId, pSubplan->id.groupId); + planDebug("QID:0x%" PRIx64 " clear subplan execution node, groupId:%d", pSubplan->id.queryId, pSubplan->id.groupId); clearSubplanExecutionNode(pSubplan->pNode); } diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index f475c974cc..03145da939 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -728,7 +728,7 @@ void schFreeJobImpl(void *job) { uint64_t queryId = pJob->queryId; int64_t refId = pJob->refId; - qDebug("qid:0x%" PRIx64 " begin to free sch job, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); + qDebug("QID:0x%" PRIx64 " begin to free sch job, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); schDropJobAllTasks(pJob); @@ -775,7 +775,7 @@ void schFreeJobImpl(void *job) { taosMemoryFreeClear(pJob->userRes.execRes); taosMemoryFreeClear(pJob->fetchRes); taosMemoryFreeClear(pJob->sql); - int32_t code = tsem_destroy(&pJob->rspSem); + int32_t code = tsem_destroy(&pJob->rspSem); if (code) { qError("tsem_destroy failed, error:%s", tstrerror(code)); } @@ -786,7 +786,7 @@ void schFreeJobImpl(void *job) { schCloseJobRef(); } - qDebug("qid:0x%" PRIx64 " sch job freed, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); + qDebug("QID:0x%" PRIx64 " sch job freed, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); } int32_t schJobFetchRows(SSchJob *pJob) { @@ -797,7 +797,7 @@ int32_t schJobFetchRows(SSchJob *pJob) { if (schChkCurrentOp(pJob, SCH_OP_FETCH, true)) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - code = tsem_wait(&pJob->rspSem); + code = tsem_wait(&pJob->rspSem); if (code) { qError("tsem_wait for fetch rspSem failed, error:%s", tstrerror(code)); SCH_RET(code); @@ -821,7 +821,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { int64_t refId = -1; SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); if (NULL == pJob) { - qError("qid:0x%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); + qError("QID:0x%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); SCH_ERR_JRET(terrno); } @@ -831,7 +831,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { if (pReq->sql) { pJob->sql = taosStrdup(pReq->sql); if (NULL == pJob->sql) { - qError("qid:0x%" PRIx64 " strdup sql %s failed", pReq->pDag->queryId, pReq->sql); + qError("QID:0x%" PRIx64 " strdup sql %s failed", pReq->pDag->queryId, pReq->sql); SCH_ERR_JRET(terrno); } } @@ -839,7 +839,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { if (pReq->allocatorRefId > 0) { pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId); if (pJob->allocatorRefId <= 0) { - qError("qid:0x%" PRIx64 " nodesMakeAllocatorWeakRef failed", pReq->pDag->queryId); + qError("QID:0x%" PRIx64 " nodesMakeAllocatorWeakRef failed", pReq->pDag->queryId); SCH_ERR_JRET(terrno); } } @@ -851,11 +851,11 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->pWorkerCb = pReq->pWorkerCb; if (pReq->pNodeList == NULL || taosArrayGetSize(pReq->pNodeList) <= 0) { - qDebug("qid:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId); + qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId); } else { pJob->nodeList = taosArrayDup(pReq->pNodeList, NULL); if (NULL == pJob->nodeList) { - qError("qid:0x%" PRIx64 " taosArrayDup failed, origNum:%d", pReq->pDag->queryId, + qError("QID:0x%" PRIx64 " taosArrayDup failed, origNum:%d", pReq->pDag->queryId, (int32_t)taosArrayGetSize(pReq->pNodeList)); SCH_ERR_JRET(terrno); } @@ -918,7 +918,7 @@ _return: int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq) { int32_t code = 0; - qDebug("qid:0x%" PRIx64 " sch job refId 0x%" PRIx64 " started", pReq->pDag->queryId, pJob->refId); + qDebug("QID:0x%" PRIx64 " sch job refId 0x%" PRIx64 " started", pReq->pDag->queryId, pJob->refId); SCH_ERR_RET(schLaunchJob(pJob)); @@ -926,7 +926,7 @@ int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); code = tsem_wait(&pJob->rspSem); if (code) { - qError("qid:0x%" PRIx64 " tsem_wait sync rspSem failed, error:%s", pReq->pDag->queryId, tstrerror(code)); + qError("QID:0x%" PRIx64 " tsem_wait sync rspSem failed, error:%s", pReq->pDag->queryId, tstrerror(code)); SCH_ERR_RET(code); } } @@ -1191,7 +1191,7 @@ int32_t schProcessOnCbBegin(SSchJob **job, SSchTask **task, uint64_t qId, int64_ (void)schAcquireJob(rId, &pJob); if (NULL == pJob) { - qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, qId, tId, rId); + qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, qId, tId, rId); SCH_ERR_RET(TSDB_CODE_QRY_JOB_NOT_EXIST); } diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index fe601c6b86..b3106d8c7e 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -500,7 +500,7 @@ _return: int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); // called if drop task rsp received code (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error @@ -513,7 +513,7 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { int32_t schHandleNotifyCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", pParam->queryId, pParam->taskId, + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); if (pMsg) { taosMemoryFree(pMsg->pData); diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 375ad5fa37..4c609fa5e2 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -996,7 +996,7 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) { int32_t code = 0; - qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, pStatus->taskId, + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, pStatus->taskId, pStatus->execId, jobTaskStatusStr(pStatus->status)); if (schProcessOnCbBegin(&pJob, &pTask, pStatus->queryId, pStatus->refId, pStatus->taskId)) { @@ -1043,12 +1043,12 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { continue; } - qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId); + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId); pJob = NULL; (void)schAcquireJob(localRsp->rId, &pJob); if (NULL == pJob) { - qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, + qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, localRsp->tId, localRsp->rId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_NOT_EXIST); } @@ -1068,7 +1068,7 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { (void)schReleaseJob(pJob->refId); - qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, localRsp->tId, code); SCH_ERR_JRET(code); diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 2688617823..75bcc326b3 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -108,7 +108,7 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) { pRange->range.maxVer, pWindow->skey, pWindow->ekey, req.reqId); code = streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, - &pTask->outputInfo.fixedDispatcher.epSet); + &pTask->outputInfo.fixedDispatcher.epSet); } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { streamTaskStartMonitorCheckRsp(pTask); @@ -171,14 +171,14 @@ void streamTaskProcessCheckMsg(SStreamMeta* pMeta, SStreamTaskCheckReq* pReq, SS streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage); SStreamTaskState pState = streamTaskGetStatus(pTask); - stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(qid:0x%" PRIx64 + stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(QID:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", pTask->id.idStr, pState.name, pRsp->oldStage, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status); streamMetaReleaseTask(pMeta, pTask); } else { pRsp->status = TASK_DOWNSTREAM_NOT_READY; - stDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(qid:0x%" PRIx64 + stDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(QID:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp check_status %d", pReq->streamId, taskId, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status); } @@ -259,7 +259,8 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa void* buf = rpcMallocCont(sizeof(SMsgHead) + len); if (buf == NULL) { - stError("s-task:0x%x vgId:%d failed prepare msg, %s at line:%d code:%s", taskId, pMeta->vgId, __func__, __LINE__, tstrerror(code)); + stError("s-task:0x%x vgId:%d failed prepare msg, %s at line:%d code:%s", taskId, pMeta->vgId, __func__, __LINE__, + tstrerror(code)); return terrno; } @@ -332,7 +333,7 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) { /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void processDownstreamReadyRsp(SStreamTask* pTask) { EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST; - int32_t code = streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); + int32_t code = streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); if (code) { stError("s-task:%s failed to set event succ, code:%s", pTask->id.idStr, tstrerror(code)); } @@ -354,7 +355,7 @@ void processDownstreamReadyRsp(SStreamTask* pTask) { stDebug("s-task:%s level:%d initial status is %s from mnode, set it to be halt", pTask->id.idStr, pTask->info.taskLevel, streamTaskGetStatusStr(pTask->status.taskStatus)); code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT); - if (code != 0) { // todo: handle error + if (code != 0) { // todo: handle error stError("s-task:%s failed to handle halt event, code:%s", pTask->id.idStr, tstrerror(code)); } } @@ -373,8 +374,9 @@ void processDownstreamReadyRsp(SStreamTask* pTask) { int32_t addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { int32_t vgId = pTask->pMeta->vgId; - int32_t code = 0;; - bool existed = false; + int32_t code = 0; + ; + bool existed = false; streamMutexLock(&pTask->lock); From 80cdeae67fcabf22b5b2f9e4285dfabe2c688b73 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Mon, 30 Sep 2024 09:53:31 +0800 Subject: [PATCH 03/61] fix build error --- include/os/osString.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/os/osString.h b/include/os/osString.h index 5f211ad2ee..30bfd61b62 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -51,7 +51,12 @@ typedef enum { M2C = 0, C2M } ConvType; #define strtod STR_TO_LD_FUNC_TAOS_FORBID #define strtold STR_TO_D_FUNC_TAOS_FORBID #define strtof STR_TO_F_FUNC_TAOS_FORBID + +#ifdef strndup +#undef strndup +#endif #define strndup STR_TO_F_FUNC_TAOS_FORBID + #endif #define tstrncpy(dst, src, size) \ From 296f2ec7769b683abd4e7ade9b89c614203b86c5 Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Sun, 29 Sep 2024 18:45:43 +0800 Subject: [PATCH 04/61] fix:[TD-32419] Free memory allocated in sclWalkCaseWhenList. --- source/libs/scalar/src/scalar.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 4f08c93c1e..2a4951d237 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -628,8 +628,8 @@ int32_t sclWalkCaseWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell cell = cell->pNext) { pWhenThen = (SWhenThenNode *)node; - SCL_ERR_RET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); - SCL_ERR_RET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); SCL_ERR_JRET(vectorCompareImpl(pCase, pWhen, pComp, rowIdx, 1, TSDB_ORDER_ASC, OP_TYPE_EQUAL)); @@ -646,6 +646,10 @@ int32_t sclWalkCaseWhenList(SScalarCtx *ctx, SNodeList *pList, struct SListCell goto _return; } + sclFreeParam(pWhen); + sclFreeParam(pThen); + taosMemoryFreeClear(pWhen); + taosMemoryFreeClear(pThen); } if (pElse) { @@ -672,8 +676,8 @@ _return: sclFreeParam(pWhen); sclFreeParam(pThen); - taosMemoryFree(pWhen); - taosMemoryFree(pThen); + taosMemoryFreeClear(pWhen); + taosMemoryFreeClear(pThen); SCL_RET(code); } From c0340388c5c2ea6cb4b8239be198e7bac5e87d7f Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Mon, 30 Sep 2024 11:36:58 +0800 Subject: [PATCH 05/61] calculate stream fill row buff size --- source/libs/executor/src/streamfilloperator.c | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index 291cc3b67b..826220581a 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -1165,12 +1165,12 @@ _end: return code; } -static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { - pFillSup->rowSize = sizeof(SResultCellData) * pFillSup->numOfAllCols; - for (int i = 0; i < pFillSup->numOfAllCols; i++) { - SFillColInfo* pCol = &pFillSup->pAllColInfo[i]; - SResSchema* pSchema = &pCol->pExpr->base.resSchema; - pFillSup->rowSize += pSchema->bytes; +static int32_t initResultBuf(SSDataBlock* pInputRes, SStreamFillSupporter* pFillSup) { + int32_t numOfCols = taosArrayGetSize(pInputRes->pDataBlock); + pFillSup->rowSize = sizeof(SResultCellData) * numOfCols; + for (int i = 0; i < numOfCols; i++) { + SColumnInfoData* pCol = taosArrayGet(pInputRes->pDataBlock, i); + pFillSup->rowSize += pCol->info.bytes; } pFillSup->next.key = INT64_MIN; pFillSup->nextNext.key = INT64_MIN; @@ -1185,7 +1185,7 @@ static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { } static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval, - SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) { + SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI, SSDataBlock* pInputRes) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter)); @@ -1214,7 +1214,7 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod pFillSup->interval = *pInterval; pFillSup->pAPI = pAPI; - code = initResultBuf(pFillSup); + code = initResultBuf(pInputRes, pFillSup); QUERY_CHECK_CODE(code, lino, _end); SExprInfo* noFillExpr = NULL; @@ -1371,7 +1371,11 @@ int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysi code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore); QUERY_CHECK_CODE(code, lino, _error); - pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI); + pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); + QUERY_CHECK_NULL(pInfo->pSrcBlock, code, lino, _error, terrno); + + pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI, + pInfo->pSrcBlock); if (!pInfo->pFillSup) { code = TSDB_CODE_FAILED; QUERY_CHECK_CODE(code, lino, _error); @@ -1380,8 +1384,7 @@ int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysi initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); QUERY_CHECK_NULL(pInfo->pRes, code, lino, _error, terrno); - pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); - QUERY_CHECK_NULL(pInfo->pSrcBlock, code, lino, _error, terrno); + code = blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); QUERY_CHECK_CODE(code, lino, _error); From efbe5746849b9a471e45073d37b2a6ed34980ae2 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Mon, 30 Sep 2024 13:58:28 +0800 Subject: [PATCH 06/61] fix: mem free --- source/dnode/vnode/src/meta/metaQuery.c | 1 + source/libs/executor/src/executil.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 14207e7fb3..484c5c0a16 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1523,6 +1523,7 @@ int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *pUidTagInfo) { } memcpy(info.pTagVal, pCur->pVal, pCur->vLen); if (taosArrayPush(pUidTagInfo, &info) == NULL) { + taosMemoryFreeClear(info.pTagVal); metaCloseCtbCursor(pCur); taosHashCleanup(pSepecifiedUidMap); return terrno; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 04b3a83264..c74aef3992 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1143,11 +1143,11 @@ SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, S varDataSetLen(tmp, tagVal.nData); memcpy(tmp + VARSTR_HEADER_SIZE, tagVal.pData, tagVal.nData); code = colDataSetVal(pColInfo, i, tmp, false); - QUERY_CHECK_CODE(code, lino, _end); #if TAG_FILTER_DEBUG qDebug("tagfilter varch:%s", tmp + 2); #endif taosMemoryFree(tmp); + QUERY_CHECK_CODE(code, lino, _end); } else { code = colDataSetVal(pColInfo, i, (const char*)&tagVal.i64, false); QUERY_CHECK_CODE(code, lino, _end); From 553b57f268aa34d1d813dae6de85da80f06a898e Mon Sep 17 00:00:00 2001 From: sheyanjie-qq <249478495@qq.com> Date: Tue, 1 Oct 2024 10:33:00 +0800 Subject: [PATCH 07/61] mod java code comments --- .../JDBC/SpringJdbcTemplate/readme.md | 16 ++--- .../src/main/resources/log4j.properties | 10 ++-- .../src/main/resources/proxool.xml | 21 +++---- .../src/main/java/com/taosdata/Worker.java | 9 +-- docs/examples/JDBC/mybatisplus-demo/readme | 8 +-- docs/examples/JDBC/springbootdemo/readme.md | 14 ++--- docs/examples/JDBC/taosdemo/readme.md | 7 ++- .../taosdemo/TaosDemoApplication.java | 15 +++-- .../taosdata/taosdemo/dao/SubTableMapper.java | 15 ++--- .../taosdemo/dao/SuperTableMapper.java | 22 +++---- .../taosdata/taosdemo/dao/TableMapper.java | 13 ++-- .../taosdemo/service/DatabaseService.java | 4 +- .../taosdemo/service/SubTableService.java | 60 ++++++++++--------- .../taosdemo/service/SuperTableService.java | 2 +- .../taosdemo/service/TableService.java | 5 +- .../service/data/FieldValueGenerator.java | 10 +++- .../service/data/SubTableMetaGenerator.java | 2 +- .../service/data/SuperTableMetaGenerator.java | 13 ++-- .../service/data/TagValueGenerator.java | 2 +- .../taosdemo/utils/TimeStampUtil.java | 6 +- .../src/main/resources/log4j.properties | 10 ++-- 21 files changed, 142 insertions(+), 122 deletions(-) diff --git a/docs/examples/JDBC/SpringJdbcTemplate/readme.md b/docs/examples/JDBC/SpringJdbcTemplate/readme.md index f59bcdbeb5..0e9812385a 100644 --- a/docs/examples/JDBC/SpringJdbcTemplate/readme.md +++ b/docs/examples/JDBC/SpringJdbcTemplate/readme.md @@ -1,11 +1,11 @@ ## TDengine Spring JDBC Template Demo -`Spring JDBC Template` 简化了原生 JDBC Connection 获取释放等操作，使得操作数据库更加方便。 +`Spring JDBC Template` simplifies the operations of acquiring and releasing native JDBC Connections, making database operations more convenient. -### 配置 +### Configuration -修改 `src/main/resources/applicationContext.xml` 文件中 TDengine 的配置信息： +Modify the TDengine configuration in the `src/main/resources/applicationContext.xml` file: ```xml @@ -20,13 +20,15 @@ ``` -### 打包运行 +### Package and run + +Navigate to the `TDengine/tests/examples/JDBC/SpringJdbcTemplate` directory and execute the following commands to generate an executable jar file. -进入 `TDengine/tests/examples/JDBC/SpringJdbcTemplate` 目录下，执行以下命令可以生成可执行 jar 包。 ```shell mvn clean package ``` -打包成功之后，进入 `target/` 目录下，执行以下命令就可运行测试： +After successfully packaging, navigate to the `target/` directory and execute the following commands to run the tests: + ```shell java -jar target/SpringJdbcTemplate-1.0-SNAPSHOT-jar-with-dependencies.jar -``` \ No newline at end of file +``` diff --git a/docs/examples/JDBC/connectionPools/src/main/resources/log4j.properties b/docs/examples/JDBC/connectionPools/src/main/resources/log4j.properties index 1299357be3..a7f4d3d492 100644 --- a/docs/examples/JDBC/connectionPools/src/main/resources/log4j.properties +++ b/docs/examples/JDBC/connectionPools/src/main/resources/log4j.properties @@ -1,21 +1,21 @@ -### 设置### +### Settings### log4j.rootLogger=debug,stdout,DebugLog,ErrorLog -### 输出信息到控制抬 ### +### Output information to the console ### log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.Target=System.out log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n -### 输出DEBUG 级别以上的日志到=logs/debug.log +### Output logs of DEBUG level and above to logs/debug.log log4j.appender.DebugLog=org.apache.log4j.DailyRollingFileAppender log4j.appender.DebugLog.File=logs/debug.log log4j.appender.DebugLog.Append=true log4j.appender.DebugLog.Threshold=DEBUG log4j.appender.DebugLog.layout=org.apache.log4j.PatternLayout log4j.appender.DebugLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n -### 输出ERROR 级别以上的日志到=logs/error.log +### Output logs of ERROR level and above to logs/error.log log4j.appender.ErrorLog=org.apache.log4j.DailyRollingFileAppender log4j.appender.ErrorLog.File=logs/error.log log4j.appender.ErrorLog.Append=true log4j.appender.ErrorLog.Threshold=ERROR log4j.appender.ErrorLog.layout=org.apache.log4j.PatternLayout -log4j.appender.ErrorLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n \ No newline at end of file +log4j.appender.ErrorLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n diff --git a/docs/examples/JDBC/connectionPools/src/main/resources/proxool.xml b/docs/examples/JDBC/connectionPools/src/main/resources/proxool.xml index 0e2ac6368a..dcef51ffb5 100644 --- a/docs/examples/JDBC/connectionPools/src/main/resources/proxool.xml +++ b/docs/examples/JDBC/connectionPools/src/main/resources/proxool.xml @@ -1,27 +1,28 @@ + ds - + jdbc:TAOS-RS://127.0.0.1:6041/log - + com.taosdata.jdbc.rs.RestfulDriver - + - + 100 - + 100 - + 1 - + 5 - + 30000 - + select server_version() - \ No newline at end of file + diff --git a/docs/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Worker.java b/docs/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Worker.java index f6e21cd729..6823ca5b4d 100644 --- a/docs/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Worker.java +++ b/docs/examples/JDBC/consumer-demo/src/main/java/com/taosdata/Worker.java @@ -35,17 +35,18 @@ public class Worker implements Runnable { public void run() { while (!Thread.interrupted()) { try { - // 控制请求频率 + // Control request rate if (semaphore.tryAcquire()) { ConsumerRecords records = consumer.poll(Duration.ofMillis(sleepTime)); pool.submit(() -> { RateLimiter limiter = RateLimiter.create(rate); try { for (ConsumerRecord record : records) { - // 流量控制 + // Traffic control limiter.acquire(); - // 业务处理数据 - System.out.println("[" + LocalDateTime.now() + "] Thread id:" + Thread.currentThread().getId() + " -> " + record.value()); + // Business data processing + System.out.println("[" + LocalDateTime.now() + "] Thread id:" + + Thread.currentThread().getId() + " -> " + record.value()); } } finally { semaphore.release(); diff --git a/docs/examples/JDBC/mybatisplus-demo/readme b/docs/examples/JDBC/mybatisplus-demo/readme index b31b6c34bf..a4816d7631 100644 --- a/docs/examples/JDBC/mybatisplus-demo/readme +++ b/docs/examples/JDBC/mybatisplus-demo/readme @@ -1,14 +1,14 @@ -# 使用说明 +# Instructions -## 创建使用db +## Create and use the database ```shell $ taos > create database mp_test ``` -## 执行测试用例 +## Execute test cases ```shell $ mvn clean test -``` \ No newline at end of file +``` diff --git a/docs/examples/JDBC/springbootdemo/readme.md b/docs/examples/JDBC/springbootdemo/readme.md index 625d43e4ed..d11bb33c83 100644 --- a/docs/examples/JDBC/springbootdemo/readme.md +++ b/docs/examples/JDBC/springbootdemo/readme.md @@ -1,6 +1,6 @@ ## TDengine SpringBoot + Mybatis Demo -## 需要提前创建 test 数据库 +## Need to create a test database in advance ``` $ taos -s 'create database if not exists test' @@ -8,7 +8,7 @@ $ taos -s 'create database if not exists test' $ curl http://localhost:8080/weather/init ``` -### 配置 application.properties +### Configure application.properties ```properties # datasource config spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver @@ -38,9 +38,9 @@ mybatis.mapper-locations=classpath:mapper/*.xml logging.level.com.taosdata.jdbc.springbootdemo.dao=debug ``` -### 主要功能 +### Main functions -* 创建数据库和表 +* Create databases and tables ```xml @@ -52,14 +52,14 @@ logging.level.com.taosdata.jdbc.springbootdemo.dao=debug ``` -* 插入单条记录 +* Insert a single record ```xml insert into test.weather (ts, temperature, humidity) values (now, #{temperature,jdbcType=INTEGER}, #{humidity,jdbcType=FLOAT}) ``` -* 插入多条记录 +* Insert multiple records ```xml @@ -69,7 +69,7 @@ logging.level.com.taosdata.jdbc.springbootdemo.dao=debug ``` -* 分页查询 +* Pagination query ```xml diff --git a/docs/examples/JDBC/taosdemo/readme.md b/docs/examples/JDBC/taosdemo/readme.md index 986eef8a05..141391d1f6 100644 --- a/docs/examples/JDBC/taosdemo/readme.md +++ b/docs/examples/JDBC/taosdemo/readme.md @@ -1,11 +1,14 @@ ``` cd tests/examples/JDBC/taosdemo mvn clean package -Dmaven.test.skip=true -# 先建表，再插入的 +# Create tables first, then insert data java -jar target/taosdemo-2.0.1-jar-with-dependencies.jar -host -database -doCreateTable true -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100 -# 不建表，直接插入的 +# Insert data directly without creating tables java -jar target/taosdemo-2.0.1-jar-with-dependencies.jar -host -database -doCreateTable false -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100 ``` 如果发生错误 Exception in thread "main" java.lang.UnsatisfiedLinkError: no taos in java.library.path 请检查是否安装 TDengine 客户端安装包或编译 TDengine 安装。如果确定已经安装过还出现这个错误，可以在命令行 java 后加 -Djava.library.path=/usr/lib 来指定寻找共享库的路径。 + + +If you encounter the error Exception in thread "main" `java.lang.UnsatisfiedLinkError: no taos in java.library.path`, please check whether the TDengine client package is installed or TDengine is compiled and installed. If you are sure it is installed and still encounter this error, you can add `-Djava.library.path=/usr/lib` after the `java` command to specify the path to the shared library. diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java index 6854054703..40d44475b8 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java @@ -24,14 +24,14 @@ public class TaosDemoApplication { private static final Logger logger = LogManager.getLogger(TaosDemoApplication.class); public static void main(String[] args) throws IOException { - // 读配置参数 + // Read configuration parameters JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(args); boolean isHelp = Arrays.asList(args).contains("--help"); if (isHelp || config.host == null || config.host.isEmpty()) { JdbcTaosdemoConfig.printHelp(); System.exit(0); } - // 初始化 + // final DataSource dataSource = DataSourceFactory.getInstance(config.host, config.port, config.user, config.password); if (config.executeSql != null && !config.executeSql.isEmpty() @@ -50,7 +50,7 @@ public class TaosDemoApplication { final SuperTableService superTableService = new SuperTableService(dataSource); final SubTableService subTableService = new SubTableService(dataSource); - // 创建数据库 + // create database long start = System.currentTimeMillis(); Map databaseParam = new HashMap<>(); databaseParam.put("database", config.database); @@ -81,13 +81,13 @@ public class TaosDemoApplication { config.prefixOfFields, config.numOfTags, config.prefixOfTags); } /**********************************************************************************/ - // 建表 + // create table start = System.currentTimeMillis(); if (config.doCreateTable) { superTableService.drop(superTableMeta.getDatabase(), superTableMeta.getName()); superTableService.create(superTableMeta); if (!config.autoCreateTable) { - // 批量建子表 + // create sub tables in batch subTableService.createSubTable(superTableMeta, config.numOfTables, config.prefixOfTable, config.numOfThreadsForCreate); } @@ -95,7 +95,7 @@ public class TaosDemoApplication { end = System.currentTimeMillis(); logger.info(">>> create table time cost : " + (end - start) + " ms."); /**********************************************************************************/ - // 插入 + // insert data long tableSize = config.numOfTables; int threadSize = config.numOfThreadsForInsert; long startTime = getProperStartTime(config.startTime, config.days); @@ -111,10 +111,9 @@ public class TaosDemoApplication { end = System.currentTimeMillis(); logger.info("insert " + affectedRows + " rows, time cost: " + (end - start) + " ms"); /**********************************************************************************/ - // 查询 /**********************************************************************************/ - // 删除表 + // drop table if (config.dropTable) { superTableService.drop(config.database, config.superTable); } diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapper.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapper.java index e0ddd220c1..013d24eb87 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapper.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapper.java @@ -9,21 +9,22 @@ import java.util.List; @Repository public interface SubTableMapper { - // 创建：子表 + // Create: SubTable void createUsingSuperTable(SubTableMeta subTableMeta); - // 插入：一张子表多个values + // Insert: Multiple records into one SubTable int insertOneTableMultiValues(SubTableValue subTableValue); - // 插入：一张子表多个values, 自动建表 + // Insert: Multiple records into one SubTable, auto create SubTables int insertOneTableMultiValuesUsingSuperTable(SubTableValue subTableValue); - // 插入：多张表多个values + // Insert: Multiple records into multiple SubTable int insertMultiTableMultiValues(List tables); - // 插入：多张表多个values，自动建表 + // Insert: Multiple records into multiple SubTable, auto create SubTables int insertMultiTableMultiValuesUsingSuperTable(List tables); - // + // -} \ No newline at end of file +} diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapper.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapper.java index 9f8cec9e8f..15cafd04fb 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapper.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapper.java @@ -6,24 +6,26 @@ import org.springframework.stereotype.Repository; @Repository public interface SuperTableMapper { - // 创建超级表 create table if not exists xxx.xxx (f1 type1, f2 type2, ... ) tags( t1 type1, t2 type2 ...) + // Create super table: create table if not exists xxx.xxx (f1 type1, f2 type2, + // ... ) tags( t1 type1, t2 type2 ...) void createSuperTable(SuperTableMeta tableMetadata); - // 删除超级表 drop table if exists xxx; + // Drop super table: drop table if exists xxx; void dropSuperTable(String database, String name); - // + // - // + // - // + // - // + // - // + // - // - - // + // + // } diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapper.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapper.java index 32d1875e4d..c0f75d2204 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapper.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapper.java @@ -9,19 +9,18 @@ import java.util.List; @Repository public interface TableMapper { - // 创建：普通表 + // Create: Normal table void create(TableMeta tableMeta); - // 插入：一张表多个value + // Insert: Multiple records into one table int insertOneTableMultiValues(TableValue values); - // 插入: 一张表多个value，指定的列 + // Insert: Multiple records into one table, specified columns int insertOneTableMultiValuesWithColumns(TableValue values); - // 插入：多个表多个value + // Insert: Multiple records into multiple tables int insertMultiTableMultiValues(List tables); - // 插入：多个表多个value, 指定的列 + // Insert: Multiple records into multiple tables, specified columns int insertMultiTableMultiValuesWithColumns(List tables); - -} \ No newline at end of file +} diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/DatabaseService.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/DatabaseService.java index 3c8e962406..68ddd78323 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/DatabaseService.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/DatabaseService.java @@ -14,12 +14,12 @@ public class DatabaseService { this.databaseMapper = new DatabaseMapperImpl(dataSource); } - // 建库，指定 name + // Create database with specified name public void createDatabase(String database) { databaseMapper.createDatabase(database); } - // 建库，指定参数 keep,days,replica等 + // Create database with specified parameters such as keep, days, replica, etc. public void createDatabase(Map map) { if (map.isEmpty()) return; diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java index b0a79dea78..690b02f065 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java @@ -27,7 +27,8 @@ public class SubTableService extends AbstractService { this.mapper = new SubTableMapperImpl(datasource); } - public void createSubTable(SuperTableMeta superTableMeta, long numOfTables, String prefixOfTable, int numOfThreadsForCreate) { + public void createSubTable(SuperTableMeta superTableMeta, long numOfTables, String prefixOfTable, + int numOfThreadsForCreate) { ExecutorService executor = Executors.newFixedThreadPool(numOfThreadsForCreate); for (long i = 0; i < numOfTables; i++) { long tableIndex = i; @@ -35,54 +36,58 @@ public class SubTableService extends AbstractService { } executor.shutdown(); try { - executor.awaitTermination(Long.MAX_VALUE,TimeUnit.NANOSECONDS); + executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); } catch (InterruptedException e) { e.printStackTrace(); } } public void createSubTable(SuperTableMeta superTableMeta, String tableName) { - // 构造数据 + // Construct data SubTableMeta meta = SubTableMetaGenerator.generate(superTableMeta, tableName); createSubTable(meta); } - // 创建一张子表，可以指定database，supertable，tablename，tag值 + // Create a sub-table, specifying database, super table, table name, and tag + // values public void createSubTable(SubTableMeta subTableMeta) { mapper.createUsingSuperTable(subTableMeta); } /*************************************************************************************************************************/ - // 插入：多线程，多表 + // Insert: Multi-threaded, multiple tables public int insert(List subTableValues, int threadSize, int frequency) { ExecutorService executor = Executors.newFixedThreadPool(threadSize); Future future = executor.submit(() -> insert(subTableValues)); executor.shutdown(); - //TODO：frequency + // TODO：frequency return getAffectRows(future); } - // 插入：单表，insert into xxx values(),()... + // Insert: Single table, insert into xxx values(),()... public int insert(SubTableValue subTableValue) { return mapper.insertOneTableMultiValues(subTableValue); } - // 插入: 多表，insert into xxx values(),()... xxx values(),()... + // Insert: Multiple tables, insert into xxx values(),()... xxx values(),()... public int insert(List subTableValues) { return mapper.insertMultiTableMultiValues(subTableValues); } - // 插入：单表，自动建表, insert into xxx using xxx tags(...) values(),()... + // Insert: Single table, auto-create table, insert into xxx using xxx tags(...) + // values(),()... public int insertAutoCreateTable(SubTableValue subTableValue) { return mapper.insertOneTableMultiValuesUsingSuperTable(subTableValue); } - // 插入：多表，自动建表, insert into xxx using XXX tags(...) values(),()... xxx using XXX tags(...) values(),()... + // Insert: Multiple tables, auto-create tables, insert into xxx using XXX + // tags(...) values(),()... xxx using XXX tags(...) values(),()... public int insertAutoCreateTable(List subTableValues) { return mapper.insertMultiTableMultiValuesUsingSuperTable(subTableValues); } - public int insertMultiThreads(SuperTableMeta superTableMeta, int threadSize, long tableSize, long startTime, long gap, JdbcTaosdemoConfig config) { + public int insertMultiThreads(SuperTableMeta superTableMeta, int threadSize, long tableSize, long startTime, + long gap, JdbcTaosdemoConfig config) { List taskList = new ArrayList<>(); List threads = IntStream.range(0, threadSize) .mapToObj(i -> { @@ -94,8 +99,7 @@ public class SubTableService extends AbstractService { startTime, config.timeGap, config.numOfRowsPerTable, config.numOfTablesPerSQL, config.numOfValuesPerSQL, config.order, config.rate, config.range, - config.prefixOfTable, config.autoCreateTable) - ); + config.prefixOfTable, config.autoCreateTable)); taskList.add(task); return new Thread(task, "InsertThread-" + i); }).collect(Collectors.toList()); @@ -126,7 +130,7 @@ public class SubTableService extends AbstractService { private class InsertTask implements Callable { private final long startTableInd; // included - private final long endTableInd; // excluded + private final long endTableInd; // excluded private final long startTime; private final long timeGap; private final long numOfRowsPerTable; @@ -140,10 +144,10 @@ public class SubTableService extends AbstractService { private final boolean autoCreateTable; public InsertTask(SuperTableMeta superTableMeta, long startTableInd, long endTableInd, - long startTime, long timeGap, - long numOfRowsPerTable, long numOfTablesPerSQL, long numOfValuesPerSQL, - int order, int rate, long range, - String prefixOfTable, boolean autoCreateTable) { + long startTime, long timeGap, + long numOfRowsPerTable, long numOfTablesPerSQL, long numOfValuesPerSQL, + int order, int rate, long range, + String prefixOfTable, boolean autoCreateTable) { this.superTableMeta = superTableMeta; this.startTableInd = startTableInd; this.endTableInd = endTableInd; @@ -159,7 +163,6 @@ public class SubTableService extends AbstractService { this.autoCreateTable = autoCreateTable; } - @Override public Integer call() { @@ -171,23 +174,27 @@ public class SubTableService extends AbstractService { int affectRows = 0; // row - for (long rowCnt = 0; rowCnt < numOfRowsPerTable; ) { + for (long rowCnt = 0; rowCnt < numOfRowsPerTable;) { long rowSize = numOfValuesPerSQL; if (rowCnt + rowSize > numOfRowsPerTable) { rowSize = numOfRowsPerTable - rowCnt; } - //table - for (long tableCnt = startTableInd; tableCnt < endTableInd; ) { + // table + for (long tableCnt = startTableInd; tableCnt < endTableInd;) { long tableSize = numOfTablesPerSQL; if (tableCnt + tableSize > endTableInd) { tableSize = endTableInd - tableCnt; } long startTime = this.startTime + rowCnt * timeGap; -// System.out.println(Thread.currentThread().getName() + " >>> " + "rowCnt: " + rowCnt + ", rowSize: " + rowSize + ", " + "tableCnt: " + tableCnt + ",tableSize: " + tableSize + ", " + "startTime: " + startTime + ",timeGap: " + timeGap + ""); + // System.out.println(Thread.currentThread().getName() + " >>> " + "rowCnt: " + + // rowCnt + ", rowSize: " + rowSize + ", " + "tableCnt: " + tableCnt + + // ",tableSize: " + tableSize + ", " + "startTime: " + startTime + ",timeGap: " + // + timeGap + ""); /***********************************************/ - // 生成数据 - List data = SubTableValueGenerator.generate(superTableMeta, prefixOfTable, tableCnt, tableSize, rowSize, startTime, timeGap); - // 乱序 + // Construct data + List data = SubTableValueGenerator.generate(superTableMeta, prefixOfTable, tableCnt, + tableSize, rowSize, startTime, timeGap); + // disorder if (order != 0) SubTableValueGenerator.disrupt(data, rate, range); // insert @@ -205,5 +212,4 @@ public class SubTableService extends AbstractService { } } - } diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SuperTableService.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SuperTableService.java index b91348e2d0..47798e0c4e 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SuperTableService.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SuperTableService.java @@ -14,7 +14,7 @@ public class SuperTableService { this.superTableMapper = new SuperTableMapperImpl(dataSource); } - // 创建超级表，指定每个field的名称和类型，每个tag的名称和类型 + // Create super table, specifying the name and type of each field and each tag public void create(SuperTableMeta superTableMeta) { superTableMapper.createSuperTable(superTableMeta); } diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java index 2504fdb0b4..2bfb963b4a 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java @@ -11,15 +11,14 @@ public class TableService extends AbstractService { private TableMapper tableMapper; - //创建一张表 + // Create a table public void create(TableMeta tableMeta) { tableMapper.create(tableMeta); } - //创建多张表 + // Create multiple tables public void create(List tables) { tables.stream().forEach(this::create); } - } diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/FieldValueGenerator.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/FieldValueGenerator.java index 73cd981a46..b3ce35231c 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/FieldValueGenerator.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/FieldValueGenerator.java @@ -11,7 +11,8 @@ public class FieldValueGenerator { public static Random random = new Random(System.currentTimeMillis()); - // 生成start到end的时间序列，时间戳为顺序，不含有乱序，field的value为随机生成 + // Generate a time series from start to end, timestamps are in order without + // disorder, field values are randomly generated public static List generate(long start, long end, long timeGap, List fieldMetaList) { List values = new ArrayList<>(); @@ -29,9 +30,12 @@ public class FieldValueGenerator { return values; } - // 生成start到end的时间序列，时间戳为顺序，含有乱序，rate为乱序的比例，range为乱序前跳范围，field的value为随机生成 + // Generate a time series from start to end, timestamps are in order but include + // disorder, rate is the proportion of disorder, range is the jump range before + // disorder, field values are randomly generated public static List disrupt(List values, int rate, long range) { - long timeGap = (long) (values.get(1).getFields().get(0).getValue()) - (long) (values.get(0).getFields().get(0).getValue()); + long timeGap = (long) (values.get(1).getFields().get(0).getValue()) + - (long) (values.get(0).getFields().get(0).getValue()); int bugSize = values.size() * rate / 100; Set bugIndSet = new HashSet<>(); while (bugIndSet.size() < bugSize) { diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableMetaGenerator.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableMetaGenerator.java index 88e3c0d26a..7bdd72ec3b 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableMetaGenerator.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableMetaGenerator.java @@ -9,7 +9,7 @@ import java.util.List; public class SubTableMetaGenerator { - // 创建tableSize张子表，使用tablePrefix作为子表名的前缀，使用superTableMeta的元数据 + // Create tableSize sub-tables, using tablePrefix as the prefix for sub-table names, and using the metadata from superTableMeta // create table xxx using XXX tags(XXX) public static List generate(SuperTableMeta superTableMeta, int tableSize, String tablePrefix) { List subTableMetaList = new ArrayList<>(); diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SuperTableMetaGenerator.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SuperTableMetaGenerator.java index 05aefd01ac..383b492c5a 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SuperTableMetaGenerator.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SuperTableMetaGenerator.java @@ -10,10 +10,11 @@ import java.util.List; public class SuperTableMetaGenerator { - // 创建超级表，使用指定SQL语句 + // Create super table using the specified SQL statement public static SuperTableMeta generate(String superTableSQL) { SuperTableMeta tableMeta = new SuperTableMeta(); - // for example : create table superTable (ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int) + // for example : create table superTable (ts timestamp, temperature float, + // humidity int) tags(location nchar(64), groupId int) superTableSQL = superTableSQL.trim().toLowerCase(); if (!superTableSQL.startsWith("create")) throw new RuntimeException("invalid create super table SQL"); @@ -54,8 +55,9 @@ public class SuperTableMetaGenerator { return tableMeta; } - // 创建超级表,指定field和tag的个数 - public static SuperTableMeta generate(String database, String name, int fieldSize, String fieldPrefix, int tagSize, String tagPrefix) { + // Create super table with specified number of fields and tags + public static SuperTableMeta generate(String database, String name, int fieldSize, String fieldPrefix, int tagSize, + String tagPrefix) { if (fieldSize < 2 || tagSize < 1) { throw new RuntimeException("create super table but fieldSize less than 2 or tagSize less than 1"); } @@ -66,7 +68,8 @@ public class SuperTableMetaGenerator { List fields = new ArrayList<>(); fields.add(new FieldMeta("ts", "timestamp")); for (int i = 1; i <= fieldSize; i++) { - fields.add(new FieldMeta(fieldPrefix + "" + i, TaosConstants.DATA_TYPES[i % TaosConstants.DATA_TYPES.length])); + fields.add( + new FieldMeta(fieldPrefix + "" + i, TaosConstants.DATA_TYPES[i % TaosConstants.DATA_TYPES.length])); } tableMetadata.setFields(fields); // tags diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/TagValueGenerator.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/TagValueGenerator.java index b8024fea45..f7b18ca7cf 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/TagValueGenerator.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/TagValueGenerator.java @@ -9,7 +9,7 @@ import java.util.List; public class TagValueGenerator { - // 创建标签值：使用tagMetas + // Create tag values using tagMetas public static List generate(List tagMetas) { List tagValues = new ArrayList<>(); for (int i = 0; i < tagMetas.size(); i++) { diff --git a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/TimeStampUtil.java b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/TimeStampUtil.java index 9cfce16d82..53748169ae 100644 --- a/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/TimeStampUtil.java +++ b/docs/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/TimeStampUtil.java @@ -41,17 +41,17 @@ public class TimeStampUtil { if (start == 0) start = now - size * timeGap; - // 如果size小于1异常 + // If size is less than 1, throw an exception if (size < 1) throw new IllegalArgumentException("size less than 1."); - // 如果timeGap为1，已经超长，需要前移start + // If timeGap is 1 and it exceeds the limit, move start forward if (start + size > now) { start = now - size; return new TimeTuple(start, now, 1); } long end = start + (long) (timeGap * size); if (end > now) { - //压缩timeGap + // Compress timeGap end = now; double gap = (end - start) / (size * 1.0f); if (gap < 1.0f) { diff --git a/docs/examples/JDBC/taosdemo/src/main/resources/log4j.properties b/docs/examples/JDBC/taosdemo/src/main/resources/log4j.properties index 352545854d..40b1478a24 100644 --- a/docs/examples/JDBC/taosdemo/src/main/resources/log4j.properties +++ b/docs/examples/JDBC/taosdemo/src/main/resources/log4j.properties @@ -1,21 +1,21 @@ -### 设置### +### Settings ### log4j.rootLogger=info,stdout -### 输出信息到控制抬 ### +### Output information to the console ### log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.Target=System.out log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n -### 输出DEBUG 级别以上的日志到=logs/debug.log +### Output logs of DEBUG level and above to logs/debug.log ### log4j.appender.DebugLog=org.apache.log4j.DailyRollingFileAppender log4j.appender.DebugLog.File=logs/debug.log log4j.appender.DebugLog.Append=true log4j.appender.DebugLog.Threshold=DEBUG log4j.appender.DebugLog.layout=org.apache.log4j.PatternLayout log4j.appender.DebugLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n -### 输出ERROR 级别以上的日志到=logs/error.log +### Output logs of ERROR level and above to logs/error.log ### log4j.appender.ErrorLog=org.apache.log4j.DailyRollingFileAppender log4j.appender.ErrorLog.File=logs/error.log log4j.appender.ErrorLog.Append=true log4j.appender.ErrorLog.Threshold=ERROR log4j.appender.ErrorLog.layout=org.apache.log4j.PatternLayout -log4j.appender.ErrorLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n \ No newline at end of file +log4j.appender.ErrorLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n From bc8520363d35bc173c3af699888a93e2157482f3 Mon Sep 17 00:00:00 2001 From: sheyanjie-qq <249478495@qq.com> Date: Tue, 1 Oct 2024 10:44:52 +0800 Subject: [PATCH 08/61] mod java code comments --- source/libs/stream/src/streamMsg.c | 855 ----------------------------- 1 file changed, 855 deletions(-) delete mode 100644 source/libs/stream/src/streamMsg.c diff --git a/source/libs/stream/src/streamMsg.c b/source/libs/stream/src/streamMsg.c deleted file mode 100644 index 1c512888e7..0000000000 --- a/source/libs/stream/src/streamMsg.c +++ /dev/null @@ -1,855 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "streamMsg.h" -#include "os.h" -#include "tstream.h" -#include "streamInt.h" - -int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo) { - TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pInfo->taskId)); - TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pInfo->nodeId)); - TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pInfo->childId)); - TAOS_CHECK_RETURN(tEncodeSEpSet(pEncoder, &pInfo->epSet)); - TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pInfo->stage)); - return 0; -} - -int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamUpstreamEpInfo* pInfo) { - TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pInfo->taskId)); - TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pInfo->nodeId)); - TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pInfo->childId)); - TAOS_CHECK_RETURN(tDecodeSEpSet(pDecoder, &pInfo->epSet)); - TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pInfo->stage)); - return 0; -} - -int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) { - TAOS_CHECK_RETURN(tStartEncode(pEncoder)); - TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pReq->streamId)); - TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pReq->checkpointId)); - TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pReq->taskId)); - TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pReq->nodeId)); - TAOS_CHECK_RETURN(tEncodeSEpSet(pEncoder, &pReq->mgmtEps)); - TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pReq->mnodeId)); - TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pReq->expireTime)); - TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pReq->transId)); - TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pReq->mndTrigger)); - tEndEncode(pEncoder); - return pEncoder->pos; -} - -int32_t tDecodeStreamCheckpointSourceReq(SDecoder* pDecoder, SStreamCheckpointSourceReq* pReq) { - TAOS_CHECK_RETURN(tStartDecode(pDecoder)); - TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pReq->streamId)); - TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pReq->checkpointId)); - TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pReq->taskId)); - TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pReq->nodeId)); - TAOS_CHECK_RETURN(tDecodeSEpSet(pDecoder, &pReq->mgmtEps)); - TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pReq->mnodeId)); - TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pReq->expireTime)); - TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pReq->transId)); - TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pReq->mndTrigger)); - tEndDecode(pDecoder); - return 0; -} - -int32_t tEncodeStreamCheckpointSourceRsp(SEncoder* pEncoder, const SStreamCheckpointSourceRsp* pRsp) { - TAOS_CHECK_RETURN(tStartEncode(pEncoder)); - TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pRsp->streamId)); - TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pRsp->checkpointId)); - TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pRsp->taskId)); - TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pRsp->nodeId)); - TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pRsp->expireTime)); - TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pRsp->success)); - tEndEncode(pEncoder); - return pEncoder->pos; -} - -int32_t tEncodeStreamTaskUpdateMsg(SEncoder* pEncoder, const SStreamTaskNodeUpdateMsg* pMsg) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartEncode(pEncoder)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pMsg->streamId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pMsg->taskId)); - - int32_t size = taosArrayGetSize(pMsg->pNodeList); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, size)); - - for (int32_t i = 0; i < size; ++i) { - SNodeUpdateInfo* pInfo = taosArrayGet(pMsg->pNodeList, i); - if (pInfo == NULL) { - TAOS_CHECK_EXIT(terrno); - } - - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pInfo->nodeId)); - TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pInfo->prevEp)); - TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pInfo->newEp)); - } - - // todo this new attribute will be result in being incompatible with previous version - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pMsg->transId)); - tEndEncode(pEncoder); -_exit: - if (code) { - return code; - } else { - return pEncoder->pos; - } -} - -int32_t tDecodeStreamTaskUpdateMsg(SDecoder* pDecoder, SStreamTaskNodeUpdateMsg* pMsg) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartDecode(pDecoder)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pMsg->streamId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pMsg->taskId)); - - int32_t size = 0; - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &size)); - pMsg->pNodeList = taosArrayInit(size, sizeof(SNodeUpdateInfo)); - if (pMsg->pNodeList == NULL) { - TAOS_CHECK_EXIT(terrno); - } - for (int32_t i = 0; i < size; ++i) { - SNodeUpdateInfo info = {0}; - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &info.nodeId)); - TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &info.prevEp)); - TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &info.newEp)); - - if (taosArrayPush(pMsg->pNodeList, &info) == NULL) { - TAOS_CHECK_EXIT(terrno); - } - } - - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pMsg->transId)); - - tEndDecode(pDecoder); -_exit: - return code; -} - -int32_t tEncodeStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartEncode(pEncoder)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->reqId)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamNodeId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamTaskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->downstreamNodeId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->downstreamTaskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->childId)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->stage)); - tEndEncode(pEncoder); - -_exit: - if (code) { - return code; - } else { - return pEncoder->pos; - } -} - -int32_t tDecodeStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartDecode(pDecoder)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->reqId)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->streamId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->upstreamNodeId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->upstreamTaskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->downstreamNodeId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->downstreamTaskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->childId)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->stage)); - tEndDecode(pDecoder); - -_exit: - return code; -} - -int32_t tEncodeStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartEncode(pEncoder)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pRsp->reqId)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pRsp->streamId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->upstreamNodeId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->upstreamTaskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->downstreamNodeId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->downstreamTaskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->childId)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pRsp->oldStage)); - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pRsp->status)); - tEndEncode(pEncoder); - -_exit: - if (code) { - return code; - } else { - return pEncoder->pos; - } -} - -int32_t tDecodeStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartDecode(pDecoder)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pRsp->reqId)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pRsp->streamId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->upstreamNodeId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->upstreamTaskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->downstreamNodeId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->downstreamTaskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->childId)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pRsp->oldStage)); - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pRsp->status)); - tEndDecode(pDecoder); - -_exit: - return code; -} - -int32_t tEncodeStreamCheckpointReadyMsg(SEncoder* pEncoder, const SStreamCheckpointReadyMsg* pReq) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartEncode(pEncoder)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->checkpointId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->downstreamTaskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->downstreamNodeId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamTaskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamNodeId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->childId)); - tEndEncode(pEncoder); - -_exit: - if (code) { - return code; - } else { - return pEncoder->pos; - } -} - -int32_t tDecodeStreamCheckpointReadyMsg(SDecoder* pDecoder, SStreamCheckpointReadyMsg* pRsp) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartDecode(pDecoder)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pRsp->streamId)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pRsp->checkpointId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->downstreamTaskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->downstreamNodeId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->upstreamTaskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->upstreamNodeId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->childId)); - tEndDecode(pDecoder); - -_exit: - return code; -} - -int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* pReq) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartEncode(pEncoder)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->stage)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->msgId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->srcVgId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->type)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->taskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->type)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamTaskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamChildId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamNodeId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamRelTaskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->blockNum)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->totalLen)); - - if (taosArrayGetSize(pReq->data) != pReq->blockNum || taosArrayGetSize(pReq->dataLen) != pReq->blockNum) { - stError("invalid dispatch req msg"); - TAOS_CHECK_EXIT(TSDB_CODE_INVALID_MSG); - } - - for (int32_t i = 0; i < pReq->blockNum; i++) { - int32_t* pLen = taosArrayGet(pReq->dataLen, i); - void* data = taosArrayGetP(pReq->data, i); - if (data == NULL || pLen == NULL) { - TAOS_CHECK_EXIT(terrno); - } - - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, *pLen)); - TAOS_CHECK_EXIT(tEncodeBinary(pEncoder, data, *pLen)); - } - tEndEncode(pEncoder); -_exit: - if (code) { - return code; - } else { - return pEncoder->pos; - } -} - -int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartDecode(pDecoder)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->stage)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->msgId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->srcVgId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->type)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->streamId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->taskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->type)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->upstreamTaskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->upstreamChildId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->upstreamNodeId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->upstreamRelTaskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->blockNum)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->totalLen)); - - if ((pReq->data = taosArrayInit(pReq->blockNum, sizeof(void*))) == NULL) { - TAOS_CHECK_EXIT(terrno); - } - if ((pReq->dataLen = taosArrayInit(pReq->blockNum, sizeof(int32_t))) == NULL) { - TAOS_CHECK_EXIT(terrno); - } - for (int32_t i = 0; i < pReq->blockNum; i++) { - int32_t len1; - uint64_t len2; - void* data; - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &len1)); - TAOS_CHECK_EXIT(tDecodeBinaryAlloc(pDecoder, &data, &len2)); - - if (len1 != len2) { - TAOS_CHECK_EXIT(TSDB_CODE_INVALID_MSG); - } - - if (taosArrayPush(pReq->dataLen, &len1) == NULL) { - TAOS_CHECK_EXIT(terrno); - } - - if (taosArrayPush(pReq->data, &data) == NULL) { - TAOS_CHECK_EXIT(terrno); - } - } - - tEndDecode(pDecoder); -_exit: - return code; -} - -void tCleanupStreamDispatchReq(SStreamDispatchReq* pReq) { - taosArrayDestroyP(pReq->data, taosMemoryFree); - taosArrayDestroy(pReq->dataLen); -} - -int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartEncode(pEncoder)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->reqId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->dstNodeId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->dstTaskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->srcNodeId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->srcTaskId)); - TAOS_CHECK_EXIT(tEncodeBinary(pEncoder, (const uint8_t*)pReq->pRetrieve, pReq->retrieveLen)); - tEndEncode(pEncoder); - -_exit: - if (code) { - return code; - } else { - return pEncoder->pos; - } -} - -int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartDecode(pDecoder)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->streamId)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->reqId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->dstNodeId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->dstTaskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->srcNodeId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->srcTaskId)); - uint64_t len = 0; - TAOS_CHECK_EXIT(tDecodeBinaryAlloc(pDecoder, (void**)&pReq->pRetrieve, &len)); - pReq->retrieveLen = (int32_t)len; - tEndDecode(pDecoder); - -_exit: - return code; -} - -void tCleanupStreamRetrieveReq(SStreamRetrieveReq* pReq) { taosMemoryFree(pReq->pRetrieve); } - -int32_t tEncodeStreamTaskCheckpointReq(SEncoder* pEncoder, const SStreamTaskCheckpointReq* pReq) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartEncode(pEncoder)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->taskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->nodeId)); - tEndEncode(pEncoder); - -_exit: - return code; -} - -int32_t tDecodeStreamTaskCheckpointReq(SDecoder* pDecoder, SStreamTaskCheckpointReq* pReq) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartDecode(pDecoder)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->streamId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->taskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->nodeId)); - tEndDecode(pDecoder); - -_exit: - return code; -} - -int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartEncode(pEncoder)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->vgId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->numOfTasks)); - - for (int32_t i = 0; i < pReq->numOfTasks; ++i) { - STaskStatusEntry* ps = taosArrayGet(pReq->pTaskStatus, i); - if (ps == NULL) { - TAOS_CHECK_EXIT(terrno); - } - - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->id.streamId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, ps->id.taskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, ps->status)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->stage)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, ps->nodeId)); - TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->inputQUsed)); - TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->inputRate)); - TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->procsTotal)); - TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->procsThroughput)); - TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->outputTotal)); - TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->outputThroughput)); - TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->sinkQuota)); - TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->sinkDataSize)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->processedVer)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->verRange.minVer)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->verRange.maxVer)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->checkpointInfo.activeId)); - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, ps->checkpointInfo.failed)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, ps->checkpointInfo.activeTransId)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->checkpointInfo.latestId)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->checkpointInfo.latestVer)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->checkpointInfo.latestTime)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->checkpointInfo.latestSize)); - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, ps->checkpointInfo.remoteBackup)); - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, ps->checkpointInfo.consensusChkptId)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->checkpointInfo.consensusTs)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->startTime)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->startCheckpointId)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->startCheckpointVer)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->hTaskId)); - } - - int32_t numOfVgs = taosArrayGetSize(pReq->pUpdateNodes); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, numOfVgs)); - - for (int j = 0; j < numOfVgs; ++j) { - int32_t* pVgId = taosArrayGet(pReq->pUpdateNodes, j); - if (pVgId == NULL) { - TAOS_CHECK_EXIT(terrno); - } - - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, *pVgId)); - } - - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->msgId)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->ts)); - tEndEncode(pEncoder); - -_exit: - if (code) { - return code; - } else { - return pEncoder->pos; - } -} - -int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartDecode(pDecoder)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->vgId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->numOfTasks)); - - if ((pReq->pTaskStatus = taosArrayInit(pReq->numOfTasks, sizeof(STaskStatusEntry))) == NULL) { - TAOS_CHECK_EXIT(terrno); - } - for (int32_t i = 0; i < pReq->numOfTasks; ++i) { - int32_t taskId = 0; - STaskStatusEntry entry = {0}; - - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.id.streamId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &taskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &entry.status)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.stage)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &entry.nodeId)); - TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.inputQUsed)); - TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.inputRate)); - TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.procsTotal)); - TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.procsThroughput)); - TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.outputTotal)); - TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.outputThroughput)); - TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.sinkQuota)); - TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.sinkDataSize)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.processedVer)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.verRange.minVer)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.verRange.maxVer)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.checkpointInfo.activeId)); - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &entry.checkpointInfo.failed)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &entry.checkpointInfo.activeTransId)); - - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.checkpointInfo.latestId)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.checkpointInfo.latestVer)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.checkpointInfo.latestTime)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.checkpointInfo.latestSize)); - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &entry.checkpointInfo.remoteBackup)); - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &entry.checkpointInfo.consensusChkptId)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.checkpointInfo.consensusTs)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.startTime)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.startCheckpointId)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.startCheckpointVer)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.hTaskId)); - - entry.id.taskId = taskId; - if (taosArrayPush(pReq->pTaskStatus, &entry) == NULL) { - TAOS_CHECK_EXIT(terrno); - } - } - - int32_t numOfVgs = 0; - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &numOfVgs)); - - if ((pReq->pUpdateNodes = taosArrayInit(numOfVgs, sizeof(int32_t))) == NULL) { - TAOS_CHECK_EXIT(terrno); - } - - for (int j = 0; j < numOfVgs; ++j) { - int32_t vgId = 0; - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &vgId)); - if (taosArrayPush(pReq->pUpdateNodes, &vgId) == NULL) { - TAOS_CHECK_EXIT(terrno); - } - } - - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->msgId)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->ts)); - tEndDecode(pDecoder); - -_exit: - return code; -} - -void tCleanupStreamHbMsg(SStreamHbMsg* pMsg) { - if (pMsg == NULL) { - return; - } - - if (pMsg->pUpdateNodes != NULL) { - taosArrayDestroy(pMsg->pUpdateNodes); - pMsg->pUpdateNodes = NULL; - } - - if (pMsg->pTaskStatus != NULL) { - taosArrayDestroy(pMsg->pTaskStatus); - pMsg->pTaskStatus = NULL; - } - - pMsg->msgId = -1; - pMsg->vgId = -1; - pMsg->numOfTasks = -1; -} - -int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartEncode(pEncoder)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->ver)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->id.streamId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->id.taskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.totalLevel)); - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->info.taskLevel)); - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->outputInfo.type)); - TAOS_CHECK_EXIT(tEncodeI16(pEncoder, pTask->msgInfo.msgType)); - - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->status.taskStatus)); - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->status.schedStatus)); - - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.selfChildId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.nodeId)); - TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->info.epSet)); - TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->info.mnodeEpset)); - - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->chkInfo.checkpointId)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->chkInfo.checkpointVer)); - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->info.fillHistory)); - - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->hTaskInfo.id.streamId)); - int32_t taskId = pTask->hTaskInfo.id.taskId; - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, taskId)); - - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->streamTaskId.streamId)); - taskId = pTask->streamTaskId.taskId; - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, taskId)); - - TAOS_CHECK_EXIT(tEncodeU64(pEncoder, pTask->dataRange.range.minVer)); - TAOS_CHECK_EXIT(tEncodeU64(pEncoder, pTask->dataRange.range.maxVer)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->dataRange.window.skey)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->dataRange.window.ekey)); - - int32_t epSz = taosArrayGetSize(pTask->upstreamInfo.pList); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, epSz)); - for (int32_t i = 0; i < epSz; i++) { - SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); - TAOS_CHECK_EXIT(tEncodeStreamEpInfo(pEncoder, pInfo)); - } - - if (pTask->info.taskLevel != TASK_LEVEL__SINK) { - TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->exec.qmsg)); - } - - if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->outputInfo.tbSink.stbUid)); - TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->outputInfo.tbSink.stbFullName)); - TAOS_CHECK_EXIT(tEncodeSSchemaWrapper(pEncoder, pTask->outputInfo.tbSink.pSchemaWrapper)); - } else if (pTask->outputInfo.type == TASK_OUTPUT__SMA) { - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->outputInfo.smaSink.smaId)); - } else if (pTask->outputInfo.type == TASK_OUTPUT__FETCH) { - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->outputInfo.fetchSink.reserved)); - } else if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->outputInfo.fixedDispatcher.taskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->outputInfo.fixedDispatcher.nodeId)); - TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->outputInfo.fixedDispatcher.epSet)); - } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { - TAOS_CHECK_EXIT(tSerializeSUseDbRspImp(pEncoder, &pTask->outputInfo.shuffleDispatcher.dbInfo)); - TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->outputInfo.shuffleDispatcher.stbFullName)); - } - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->info.delaySchedParam)); - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->subtableWithoutMd5)); - TAOS_CHECK_EXIT(tEncodeCStrWithLen(pEncoder, pTask->reserve, sizeof(pTask->reserve) - 1)); - - tEndEncode(pEncoder); -_exit: - return code; -} - -int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { - int32_t taskId = 0; - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartDecode(pDecoder)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->ver)); - if (pTask->ver <= SSTREAM_TASK_INCOMPATIBLE_VER || pTask->ver > SSTREAM_TASK_VER) { - TAOS_CHECK_EXIT(TSDB_CODE_INVALID_MSG); - } - - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->id.streamId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->id.taskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.totalLevel)); - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->info.taskLevel)); - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->outputInfo.type)); - TAOS_CHECK_EXIT(tDecodeI16(pDecoder, &pTask->msgInfo.msgType)); - - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->status.taskStatus)); - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->status.schedStatus)); - - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.selfChildId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.nodeId)); - TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->info.epSet)); - TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->info.mnodeEpset)); - - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->chkInfo.checkpointId)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->chkInfo.checkpointVer)); - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->info.fillHistory)); - - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->hTaskInfo.id.streamId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &taskId)); - pTask->hTaskInfo.id.taskId = taskId; - - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->streamTaskId.streamId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &taskId)); - pTask->streamTaskId.taskId = taskId; - - TAOS_CHECK_EXIT(tDecodeU64(pDecoder, (uint64_t*)&pTask->dataRange.range.minVer)); - TAOS_CHECK_EXIT(tDecodeU64(pDecoder, (uint64_t*)&pTask->dataRange.range.maxVer)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->dataRange.window.skey)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->dataRange.window.ekey)); - - int32_t epSz = -1; - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &epSz) < 0); - - if ((pTask->upstreamInfo.pList = taosArrayInit(epSz, POINTER_BYTES)) == NULL) { - TAOS_CHECK_EXIT(terrno); - } - for (int32_t i = 0; i < epSz; i++) { - SStreamUpstreamEpInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamUpstreamEpInfo)); - if (pInfo == NULL) { - TAOS_CHECK_EXIT(terrno); - } - if ((code = tDecodeStreamEpInfo(pDecoder, pInfo)) < 0) { - taosMemoryFreeClear(pInfo); - goto _exit; - } - if (taosArrayPush(pTask->upstreamInfo.pList, &pInfo) == NULL) { - TAOS_CHECK_EXIT(terrno); - } - } - - if (pTask->info.taskLevel != TASK_LEVEL__SINK) { - TAOS_CHECK_EXIT(tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg)); - } - - if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->outputInfo.tbSink.stbUid)); - TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->outputInfo.tbSink.stbFullName)); - pTask->outputInfo.tbSink.pSchemaWrapper = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); - if (pTask->outputInfo.tbSink.pSchemaWrapper == NULL) { - TAOS_CHECK_EXIT(terrno); - } - TAOS_CHECK_EXIT(tDecodeSSchemaWrapper(pDecoder, pTask->outputInfo.tbSink.pSchemaWrapper)); - } else if (pTask->outputInfo.type == TASK_OUTPUT__SMA) { - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->outputInfo.smaSink.smaId)); - } else if (pTask->outputInfo.type == TASK_OUTPUT__FETCH) { - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->outputInfo.fetchSink.reserved)); - } else if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->outputInfo.fixedDispatcher.taskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->outputInfo.fixedDispatcher.nodeId)); - TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->outputInfo.fixedDispatcher.epSet)); - } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { - TAOS_CHECK_EXIT(tDeserializeSUseDbRspImp(pDecoder, &pTask->outputInfo.shuffleDispatcher.dbInfo)); - TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->outputInfo.shuffleDispatcher.stbFullName)); - } - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->info.delaySchedParam)); - if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER) { - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->subtableWithoutMd5)); - } - TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->reserve)); - - tEndDecode(pDecoder); - -_exit: - return code; -} - -int32_t tEncodeStreamTaskChkptReport(SEncoder* pEncoder, const SCheckpointReport* pReq) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartEncode(pEncoder)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->taskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->nodeId)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->checkpointId)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->checkpointVer)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->checkpointTs)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->transId)); - TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pReq->dropHTask)); - tEndEncode(pEncoder); - -_exit: - return code; -} - -int32_t tDecodeStreamTaskChkptReport(SDecoder* pDecoder, SCheckpointReport* pReq) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartDecode(pDecoder)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->streamId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->taskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->nodeId)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->checkpointId)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->checkpointVer)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->checkpointTs)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->transId)); - TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pReq->dropHTask)); - tEndDecode(pDecoder); - -_exit: - return code; -} - -int32_t tEncodeRestoreCheckpointInfo(SEncoder* pEncoder, const SRestoreCheckpointInfo* pReq) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartEncode(pEncoder)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->startTs)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId)); - TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->checkpointId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->transId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->taskId)); - TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->nodeId)); - tEndEncode(pEncoder); - -_exit: - if (code) { - return code; - } else { - return pEncoder->pos; - } -} - -int32_t tDecodeRestoreCheckpointInfo(SDecoder* pDecoder, SRestoreCheckpointInfo* pReq) { - int32_t code = 0; - int32_t lino; - - TAOS_CHECK_EXIT(tStartDecode(pDecoder)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->startTs)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->streamId)); - TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->checkpointId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->transId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->taskId)); - TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->nodeId)); - tEndDecode(pDecoder); - -_exit: - return code; -} From 7a1a68f435f55d60db5e88bb4ac5235f26f60618 Mon Sep 17 00:00:00 2001 From: sheyanjie-qq <249478495@qq.com> Date: Tue, 1 Oct 2024 10:56:25 +0800 Subject: [PATCH 09/61] recover streamMsg.c --- source/libs/stream/src/streamMsg.c | 855 +++++++++++++++++++++++++++++ 1 file changed, 855 insertions(+) create mode 100644 source/libs/stream/src/streamMsg.c diff --git a/source/libs/stream/src/streamMsg.c b/source/libs/stream/src/streamMsg.c new file mode 100644 index 0000000000..1c512888e7 --- /dev/null +++ b/source/libs/stream/src/streamMsg.c @@ -0,0 +1,855 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "streamMsg.h" +#include "os.h" +#include "tstream.h" +#include "streamInt.h" + +int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo) { + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pInfo->taskId)); + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pInfo->nodeId)); + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pInfo->childId)); + TAOS_CHECK_RETURN(tEncodeSEpSet(pEncoder, &pInfo->epSet)); + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pInfo->stage)); + return 0; +} + +int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamUpstreamEpInfo* pInfo) { + TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pInfo->taskId)); + TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pInfo->nodeId)); + TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pInfo->childId)); + TAOS_CHECK_RETURN(tDecodeSEpSet(pDecoder, &pInfo->epSet)); + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pInfo->stage)); + return 0; +} + +int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) { + TAOS_CHECK_RETURN(tStartEncode(pEncoder)); + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pReq->streamId)); + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pReq->checkpointId)); + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pReq->taskId)); + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pReq->nodeId)); + TAOS_CHECK_RETURN(tEncodeSEpSet(pEncoder, &pReq->mgmtEps)); + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pReq->mnodeId)); + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pReq->expireTime)); + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pReq->transId)); + TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pReq->mndTrigger)); + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeStreamCheckpointSourceReq(SDecoder* pDecoder, SStreamCheckpointSourceReq* pReq) { + TAOS_CHECK_RETURN(tStartDecode(pDecoder)); + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pReq->streamId)); + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pReq->checkpointId)); + TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pReq->taskId)); + TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pReq->nodeId)); + TAOS_CHECK_RETURN(tDecodeSEpSet(pDecoder, &pReq->mgmtEps)); + TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pReq->mnodeId)); + TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pReq->expireTime)); + TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pReq->transId)); + TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pReq->mndTrigger)); + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeStreamCheckpointSourceRsp(SEncoder* pEncoder, const SStreamCheckpointSourceRsp* pRsp) { + TAOS_CHECK_RETURN(tStartEncode(pEncoder)); + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pRsp->streamId)); + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pRsp->checkpointId)); + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pRsp->taskId)); + TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pRsp->nodeId)); + TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pRsp->expireTime)); + TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pRsp->success)); + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tEncodeStreamTaskUpdateMsg(SEncoder* pEncoder, const SStreamTaskNodeUpdateMsg* pMsg) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartEncode(pEncoder)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pMsg->streamId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pMsg->taskId)); + + int32_t size = taosArrayGetSize(pMsg->pNodeList); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, size)); + + for (int32_t i = 0; i < size; ++i) { + SNodeUpdateInfo* pInfo = taosArrayGet(pMsg->pNodeList, i); + if (pInfo == NULL) { + TAOS_CHECK_EXIT(terrno); + } + + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pInfo->nodeId)); + TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pInfo->prevEp)); + TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pInfo->newEp)); + } + + // todo this new attribute will be result in being incompatible with previous version + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pMsg->transId)); + tEndEncode(pEncoder); +_exit: + if (code) { + return code; + } else { + return pEncoder->pos; + } +} + +int32_t tDecodeStreamTaskUpdateMsg(SDecoder* pDecoder, SStreamTaskNodeUpdateMsg* pMsg) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartDecode(pDecoder)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pMsg->streamId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pMsg->taskId)); + + int32_t size = 0; + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &size)); + pMsg->pNodeList = taosArrayInit(size, sizeof(SNodeUpdateInfo)); + if (pMsg->pNodeList == NULL) { + TAOS_CHECK_EXIT(terrno); + } + for (int32_t i = 0; i < size; ++i) { + SNodeUpdateInfo info = {0}; + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &info.nodeId)); + TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &info.prevEp)); + TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &info.newEp)); + + if (taosArrayPush(pMsg->pNodeList, &info) == NULL) { + TAOS_CHECK_EXIT(terrno); + } + } + + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pMsg->transId)); + + tEndDecode(pDecoder); +_exit: + return code; +} + +int32_t tEncodeStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartEncode(pEncoder)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->reqId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamNodeId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamTaskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->downstreamNodeId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->downstreamTaskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->childId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->stage)); + tEndEncode(pEncoder); + +_exit: + if (code) { + return code; + } else { + return pEncoder->pos; + } +} + +int32_t tDecodeStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartDecode(pDecoder)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->reqId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->streamId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->upstreamNodeId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->upstreamTaskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->downstreamNodeId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->downstreamTaskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->childId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->stage)); + tEndDecode(pDecoder); + +_exit: + return code; +} + +int32_t tEncodeStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartEncode(pEncoder)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pRsp->reqId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pRsp->streamId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->upstreamNodeId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->upstreamTaskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->downstreamNodeId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->downstreamTaskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->childId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pRsp->oldStage)); + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pRsp->status)); + tEndEncode(pEncoder); + +_exit: + if (code) { + return code; + } else { + return pEncoder->pos; + } +} + +int32_t tDecodeStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartDecode(pDecoder)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pRsp->reqId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pRsp->streamId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->upstreamNodeId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->upstreamTaskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->downstreamNodeId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->downstreamTaskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->childId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pRsp->oldStage)); + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pRsp->status)); + tEndDecode(pDecoder); + +_exit: + return code; +} + +int32_t tEncodeStreamCheckpointReadyMsg(SEncoder* pEncoder, const SStreamCheckpointReadyMsg* pReq) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartEncode(pEncoder)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->checkpointId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->downstreamTaskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->downstreamNodeId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamTaskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamNodeId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->childId)); + tEndEncode(pEncoder); + +_exit: + if (code) { + return code; + } else { + return pEncoder->pos; + } +} + +int32_t tDecodeStreamCheckpointReadyMsg(SDecoder* pDecoder, SStreamCheckpointReadyMsg* pRsp) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartDecode(pDecoder)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pRsp->streamId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pRsp->checkpointId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->downstreamTaskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->downstreamNodeId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->upstreamTaskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->upstreamNodeId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->childId)); + tEndDecode(pDecoder); + +_exit: + return code; +} + +int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* pReq) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartEncode(pEncoder)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->stage)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->msgId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->srcVgId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->type)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->taskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->type)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamTaskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamChildId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamNodeId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->upstreamRelTaskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->blockNum)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->totalLen)); + + if (taosArrayGetSize(pReq->data) != pReq->blockNum || taosArrayGetSize(pReq->dataLen) != pReq->blockNum) { + stError("invalid dispatch req msg"); + TAOS_CHECK_EXIT(TSDB_CODE_INVALID_MSG); + } + + for (int32_t i = 0; i < pReq->blockNum; i++) { + int32_t* pLen = taosArrayGet(pReq->dataLen, i); + void* data = taosArrayGetP(pReq->data, i); + if (data == NULL || pLen == NULL) { + TAOS_CHECK_EXIT(terrno); + } + + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, *pLen)); + TAOS_CHECK_EXIT(tEncodeBinary(pEncoder, data, *pLen)); + } + tEndEncode(pEncoder); +_exit: + if (code) { + return code; + } else { + return pEncoder->pos; + } +} + +int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartDecode(pDecoder)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->stage)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->msgId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->srcVgId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->type)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->streamId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->taskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->type)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->upstreamTaskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->upstreamChildId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->upstreamNodeId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->upstreamRelTaskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->blockNum)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->totalLen)); + + if ((pReq->data = taosArrayInit(pReq->blockNum, sizeof(void*))) == NULL) { + TAOS_CHECK_EXIT(terrno); + } + if ((pReq->dataLen = taosArrayInit(pReq->blockNum, sizeof(int32_t))) == NULL) { + TAOS_CHECK_EXIT(terrno); + } + for (int32_t i = 0; i < pReq->blockNum; i++) { + int32_t len1; + uint64_t len2; + void* data; + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &len1)); + TAOS_CHECK_EXIT(tDecodeBinaryAlloc(pDecoder, &data, &len2)); + + if (len1 != len2) { + TAOS_CHECK_EXIT(TSDB_CODE_INVALID_MSG); + } + + if (taosArrayPush(pReq->dataLen, &len1) == NULL) { + TAOS_CHECK_EXIT(terrno); + } + + if (taosArrayPush(pReq->data, &data) == NULL) { + TAOS_CHECK_EXIT(terrno); + } + } + + tEndDecode(pDecoder); +_exit: + return code; +} + +void tCleanupStreamDispatchReq(SStreamDispatchReq* pReq) { + taosArrayDestroyP(pReq->data, taosMemoryFree); + taosArrayDestroy(pReq->dataLen); +} + +int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartEncode(pEncoder)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->reqId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->dstNodeId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->dstTaskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->srcNodeId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->srcTaskId)); + TAOS_CHECK_EXIT(tEncodeBinary(pEncoder, (const uint8_t*)pReq->pRetrieve, pReq->retrieveLen)); + tEndEncode(pEncoder); + +_exit: + if (code) { + return code; + } else { + return pEncoder->pos; + } +} + +int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartDecode(pDecoder)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->streamId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->reqId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->dstNodeId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->dstTaskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->srcNodeId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->srcTaskId)); + uint64_t len = 0; + TAOS_CHECK_EXIT(tDecodeBinaryAlloc(pDecoder, (void**)&pReq->pRetrieve, &len)); + pReq->retrieveLen = (int32_t)len; + tEndDecode(pDecoder); + +_exit: + return code; +} + +void tCleanupStreamRetrieveReq(SStreamRetrieveReq* pReq) { taosMemoryFree(pReq->pRetrieve); } + +int32_t tEncodeStreamTaskCheckpointReq(SEncoder* pEncoder, const SStreamTaskCheckpointReq* pReq) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartEncode(pEncoder)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->taskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->nodeId)); + tEndEncode(pEncoder); + +_exit: + return code; +} + +int32_t tDecodeStreamTaskCheckpointReq(SDecoder* pDecoder, SStreamTaskCheckpointReq* pReq) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartDecode(pDecoder)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->streamId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->taskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->nodeId)); + tEndDecode(pDecoder); + +_exit: + return code; +} + +int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartEncode(pEncoder)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->vgId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->numOfTasks)); + + for (int32_t i = 0; i < pReq->numOfTasks; ++i) { + STaskStatusEntry* ps = taosArrayGet(pReq->pTaskStatus, i); + if (ps == NULL) { + TAOS_CHECK_EXIT(terrno); + } + + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->id.streamId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, ps->id.taskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, ps->status)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->stage)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, ps->nodeId)); + TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->inputQUsed)); + TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->inputRate)); + TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->procsTotal)); + TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->procsThroughput)); + TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->outputTotal)); + TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->outputThroughput)); + TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->sinkQuota)); + TAOS_CHECK_EXIT(tEncodeDouble(pEncoder, ps->sinkDataSize)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->processedVer)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->verRange.minVer)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->verRange.maxVer)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->checkpointInfo.activeId)); + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, ps->checkpointInfo.failed)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, ps->checkpointInfo.activeTransId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->checkpointInfo.latestId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->checkpointInfo.latestVer)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->checkpointInfo.latestTime)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->checkpointInfo.latestSize)); + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, ps->checkpointInfo.remoteBackup)); + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, ps->checkpointInfo.consensusChkptId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->checkpointInfo.consensusTs)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->startTime)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->startCheckpointId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->startCheckpointVer)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, ps->hTaskId)); + } + + int32_t numOfVgs = taosArrayGetSize(pReq->pUpdateNodes); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, numOfVgs)); + + for (int j = 0; j < numOfVgs; ++j) { + int32_t* pVgId = taosArrayGet(pReq->pUpdateNodes, j); + if (pVgId == NULL) { + TAOS_CHECK_EXIT(terrno); + } + + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, *pVgId)); + } + + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->msgId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->ts)); + tEndEncode(pEncoder); + +_exit: + if (code) { + return code; + } else { + return pEncoder->pos; + } +} + +int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartDecode(pDecoder)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->vgId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->numOfTasks)); + + if ((pReq->pTaskStatus = taosArrayInit(pReq->numOfTasks, sizeof(STaskStatusEntry))) == NULL) { + TAOS_CHECK_EXIT(terrno); + } + for (int32_t i = 0; i < pReq->numOfTasks; ++i) { + int32_t taskId = 0; + STaskStatusEntry entry = {0}; + + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.id.streamId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &taskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &entry.status)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.stage)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &entry.nodeId)); + TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.inputQUsed)); + TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.inputRate)); + TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.procsTotal)); + TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.procsThroughput)); + TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.outputTotal)); + TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.outputThroughput)); + TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.sinkQuota)); + TAOS_CHECK_EXIT(tDecodeDouble(pDecoder, &entry.sinkDataSize)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.processedVer)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.verRange.minVer)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.verRange.maxVer)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.checkpointInfo.activeId)); + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &entry.checkpointInfo.failed)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &entry.checkpointInfo.activeTransId)); + + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.checkpointInfo.latestId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.checkpointInfo.latestVer)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.checkpointInfo.latestTime)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.checkpointInfo.latestSize)); + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &entry.checkpointInfo.remoteBackup)); + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &entry.checkpointInfo.consensusChkptId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.checkpointInfo.consensusTs)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.startTime)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.startCheckpointId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.startCheckpointVer)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &entry.hTaskId)); + + entry.id.taskId = taskId; + if (taosArrayPush(pReq->pTaskStatus, &entry) == NULL) { + TAOS_CHECK_EXIT(terrno); + } + } + + int32_t numOfVgs = 0; + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &numOfVgs)); + + if ((pReq->pUpdateNodes = taosArrayInit(numOfVgs, sizeof(int32_t))) == NULL) { + TAOS_CHECK_EXIT(terrno); + } + + for (int j = 0; j < numOfVgs; ++j) { + int32_t vgId = 0; + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &vgId)); + if (taosArrayPush(pReq->pUpdateNodes, &vgId) == NULL) { + TAOS_CHECK_EXIT(terrno); + } + } + + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->msgId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->ts)); + tEndDecode(pDecoder); + +_exit: + return code; +} + +void tCleanupStreamHbMsg(SStreamHbMsg* pMsg) { + if (pMsg == NULL) { + return; + } + + if (pMsg->pUpdateNodes != NULL) { + taosArrayDestroy(pMsg->pUpdateNodes); + pMsg->pUpdateNodes = NULL; + } + + if (pMsg->pTaskStatus != NULL) { + taosArrayDestroy(pMsg->pTaskStatus); + pMsg->pTaskStatus = NULL; + } + + pMsg->msgId = -1; + pMsg->vgId = -1; + pMsg->numOfTasks = -1; +} + +int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartEncode(pEncoder)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->ver)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->id.streamId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->id.taskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.totalLevel)); + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->info.taskLevel)); + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->outputInfo.type)); + TAOS_CHECK_EXIT(tEncodeI16(pEncoder, pTask->msgInfo.msgType)); + + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->status.taskStatus)); + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->status.schedStatus)); + + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.selfChildId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->info.nodeId)); + TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->info.epSet)); + TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->info.mnodeEpset)); + + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->chkInfo.checkpointId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->chkInfo.checkpointVer)); + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->info.fillHistory)); + + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->hTaskInfo.id.streamId)); + int32_t taskId = pTask->hTaskInfo.id.taskId; + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, taskId)); + + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->streamTaskId.streamId)); + taskId = pTask->streamTaskId.taskId; + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, taskId)); + + TAOS_CHECK_EXIT(tEncodeU64(pEncoder, pTask->dataRange.range.minVer)); + TAOS_CHECK_EXIT(tEncodeU64(pEncoder, pTask->dataRange.range.maxVer)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->dataRange.window.skey)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->dataRange.window.ekey)); + + int32_t epSz = taosArrayGetSize(pTask->upstreamInfo.pList); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, epSz)); + for (int32_t i = 0; i < epSz; i++) { + SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); + TAOS_CHECK_EXIT(tEncodeStreamEpInfo(pEncoder, pInfo)); + } + + if (pTask->info.taskLevel != TASK_LEVEL__SINK) { + TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->exec.qmsg)); + } + + if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->outputInfo.tbSink.stbUid)); + TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->outputInfo.tbSink.stbFullName)); + TAOS_CHECK_EXIT(tEncodeSSchemaWrapper(pEncoder, pTask->outputInfo.tbSink.pSchemaWrapper)); + } else if (pTask->outputInfo.type == TASK_OUTPUT__SMA) { + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->outputInfo.smaSink.smaId)); + } else if (pTask->outputInfo.type == TASK_OUTPUT__FETCH) { + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->outputInfo.fetchSink.reserved)); + } else if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->outputInfo.fixedDispatcher.taskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pTask->outputInfo.fixedDispatcher.nodeId)); + TAOS_CHECK_EXIT(tEncodeSEpSet(pEncoder, &pTask->outputInfo.fixedDispatcher.epSet)); + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { + TAOS_CHECK_EXIT(tSerializeSUseDbRspImp(pEncoder, &pTask->outputInfo.shuffleDispatcher.dbInfo)); + TAOS_CHECK_EXIT(tEncodeCStr(pEncoder, pTask->outputInfo.shuffleDispatcher.stbFullName)); + } + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pTask->info.delaySchedParam)); + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pTask->subtableWithoutMd5)); + TAOS_CHECK_EXIT(tEncodeCStrWithLen(pEncoder, pTask->reserve, sizeof(pTask->reserve) - 1)); + + tEndEncode(pEncoder); +_exit: + return code; +} + +int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { + int32_t taskId = 0; + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartDecode(pDecoder)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->ver)); + if (pTask->ver <= SSTREAM_TASK_INCOMPATIBLE_VER || pTask->ver > SSTREAM_TASK_VER) { + TAOS_CHECK_EXIT(TSDB_CODE_INVALID_MSG); + } + + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->id.streamId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->id.taskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.totalLevel)); + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->info.taskLevel)); + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->outputInfo.type)); + TAOS_CHECK_EXIT(tDecodeI16(pDecoder, &pTask->msgInfo.msgType)); + + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->status.taskStatus)); + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->status.schedStatus)); + + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.selfChildId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->info.nodeId)); + TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->info.epSet)); + TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->info.mnodeEpset)); + + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->chkInfo.checkpointId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->chkInfo.checkpointVer)); + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->info.fillHistory)); + + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->hTaskInfo.id.streamId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &taskId)); + pTask->hTaskInfo.id.taskId = taskId; + + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->streamTaskId.streamId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &taskId)); + pTask->streamTaskId.taskId = taskId; + + TAOS_CHECK_EXIT(tDecodeU64(pDecoder, (uint64_t*)&pTask->dataRange.range.minVer)); + TAOS_CHECK_EXIT(tDecodeU64(pDecoder, (uint64_t*)&pTask->dataRange.range.maxVer)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->dataRange.window.skey)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->dataRange.window.ekey)); + + int32_t epSz = -1; + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &epSz) < 0); + + if ((pTask->upstreamInfo.pList = taosArrayInit(epSz, POINTER_BYTES)) == NULL) { + TAOS_CHECK_EXIT(terrno); + } + for (int32_t i = 0; i < epSz; i++) { + SStreamUpstreamEpInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamUpstreamEpInfo)); + if (pInfo == NULL) { + TAOS_CHECK_EXIT(terrno); + } + if ((code = tDecodeStreamEpInfo(pDecoder, pInfo)) < 0) { + taosMemoryFreeClear(pInfo); + goto _exit; + } + if (taosArrayPush(pTask->upstreamInfo.pList, &pInfo) == NULL) { + TAOS_CHECK_EXIT(terrno); + } + } + + if (pTask->info.taskLevel != TASK_LEVEL__SINK) { + TAOS_CHECK_EXIT(tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg)); + } + + if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->outputInfo.tbSink.stbUid)); + TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->outputInfo.tbSink.stbFullName)); + pTask->outputInfo.tbSink.pSchemaWrapper = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); + if (pTask->outputInfo.tbSink.pSchemaWrapper == NULL) { + TAOS_CHECK_EXIT(terrno); + } + TAOS_CHECK_EXIT(tDecodeSSchemaWrapper(pDecoder, pTask->outputInfo.tbSink.pSchemaWrapper)); + } else if (pTask->outputInfo.type == TASK_OUTPUT__SMA) { + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->outputInfo.smaSink.smaId)); + } else if (pTask->outputInfo.type == TASK_OUTPUT__FETCH) { + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->outputInfo.fetchSink.reserved)); + } else if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->outputInfo.fixedDispatcher.taskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pTask->outputInfo.fixedDispatcher.nodeId)); + TAOS_CHECK_EXIT(tDecodeSEpSet(pDecoder, &pTask->outputInfo.fixedDispatcher.epSet)); + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { + TAOS_CHECK_EXIT(tDeserializeSUseDbRspImp(pDecoder, &pTask->outputInfo.shuffleDispatcher.dbInfo)); + TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->outputInfo.shuffleDispatcher.stbFullName)); + } + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pTask->info.delaySchedParam)); + if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER) { + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pTask->subtableWithoutMd5)); + } + TAOS_CHECK_EXIT(tDecodeCStrTo(pDecoder, pTask->reserve)); + + tEndDecode(pDecoder); + +_exit: + return code; +} + +int32_t tEncodeStreamTaskChkptReport(SEncoder* pEncoder, const SCheckpointReport* pReq) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartEncode(pEncoder)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->taskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->nodeId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->checkpointId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->checkpointVer)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->checkpointTs)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->transId)); + TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pReq->dropHTask)); + tEndEncode(pEncoder); + +_exit: + return code; +} + +int32_t tDecodeStreamTaskChkptReport(SDecoder* pDecoder, SCheckpointReport* pReq) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartDecode(pDecoder)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->streamId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->taskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->nodeId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->checkpointId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->checkpointVer)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->checkpointTs)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->transId)); + TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pReq->dropHTask)); + tEndDecode(pDecoder); + +_exit: + return code; +} + +int32_t tEncodeRestoreCheckpointInfo(SEncoder* pEncoder, const SRestoreCheckpointInfo* pReq) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartEncode(pEncoder)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->startTs)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->streamId)); + TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->checkpointId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->transId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->taskId)); + TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->nodeId)); + tEndEncode(pEncoder); + +_exit: + if (code) { + return code; + } else { + return pEncoder->pos; + } +} + +int32_t tDecodeRestoreCheckpointInfo(SDecoder* pDecoder, SRestoreCheckpointInfo* pReq) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tStartDecode(pDecoder)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->startTs)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->streamId)); + TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pReq->checkpointId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->transId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->taskId)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->nodeId)); + tEndDecode(pDecoder); + +_exit: + return code; +} From 4b0ebfb9d2cd2411caf93a5b9c9120266b1a714c Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 2 Oct 2024 22:17:08 +0800 Subject: [PATCH 10/61] update mac install script --- packaging/tools/make_install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index ea19125bf5..13447bd5e4 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -616,8 +616,8 @@ function update_TDengine() { [ -f ${installDir}/bin/taosadapter ] && \ echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}" else - echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" - echo -e "${GREEN_DARK}To start Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}" + echo -e "${GREEN_DARK}To start service ${NC}: sudo launchctl start com.tdengine.taosd${NC}" + echo -e "${GREEN_DARK}To start Adapter ${NC}: sudo launchctl start com.tdengine.taosadapter${NC}" fi fi @@ -668,8 +668,8 @@ function install_TDengine() { [ -f ${installDir}/bin/taosadapter ] && \ echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}" else - echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" - echo -e "${GREEN_DARK}To start Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}" + echo -e "${GREEN_DARK}To start service ${NC}: sudo launchctl start com.tdengine.taosd${NC}" + echo -e "${GREEN_DARK}To start Adapter ${NC}: sudo launchctl start com.tdengine.taosadapter${NC}" fi fi From d4bcd463660760e45155088691b5e44130fd8dea Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 8 Oct 2024 06:38:56 +0800 Subject: [PATCH 11/61] docs: release ver-3.3.3.0 --- cmake/cmake.version | 2 +- docs/en/28-releases/01-tdengine.md | 4 ++++ docs/zh/28-releases/01-tdengine.md | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index 3bb764612e..c600c084fd 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.3.3.0.alpha") + SET(TD_VER_NUMBER "3.3.4.0.alpha") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index a6e157cf74..486fe2c015 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -20,6 +20,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t import Release from "/components/ReleaseV3"; +## 3.3.3.0 + + + ## 3.3.2.0 diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 5b3abcb341..0f9ceada50 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -24,6 +24,10 @@ TDengine 3.x 各版本安装包下载链接如下： import Release from "/components/ReleaseV3"; +## 3.3.3.0 + + + ## 3.3.2.0 From 11252907368cada80a62e7d91b03b67167337023 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 8 Oct 2024 08:11:16 +0800 Subject: [PATCH 12/61] Update 07-explorer.md Add more details for taosExplorer installation. --- docs/zh/14-reference/01-components/07-explorer.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/14-reference/01-components/07-explorer.md b/docs/zh/14-reference/01-components/07-explorer.md index 5d17970b38..88269c502f 100644 --- a/docs/zh/14-reference/01-components/07-explorer.md +++ b/docs/zh/14-reference/01-components/07-explorer.md @@ -4,11 +4,11 @@ sidebar_label: taosExplorer toc_max_heading_level: 4 --- -taosExplorer 是一个为用户提供 TDengine 实例的可视化管理交互工具的 web 服务。本节主要讲述其安装和部署。它的各项功能都是基于简单易上手的图形界面，可以直接尝试，如果有需要也可以考高级功能和运维指南中的相关内容。为了确保访问 taosExplorer 的最佳体验，请使用 Chrome 79 及以上版本，或 Edge 79 及以上版本。 +taosExplorer 是一个为用户提供 TDengine 实例的可视化管理交互工具的 web 服务, 没有开源，但随安装包免费提供。本节主要讲述其安装和部署。它的各项功能都是基于简单易上手的图形界面，可以直接尝试，如果有需要也可以考高级功能和运维指南中的相关内容。为了确保访问 taosExplorer 的最佳体验，请使用 Chrome 79 及以上版本，或 Edge 79 及以上版本。 ## 安装 -taosEexplorer 无需单独安装，从 TDengine 3.3.0.0 版本开始，它随着 TDengine Enterprise Server 安装包一起发布，安装完成后，就可以看到 `taos-explorer` 服务。 +taosEexplorer 无需单独安装，从 TDengine 3.3.0.0 版本开始，它随着 TDengine 安装包一起发布，安装完成后，就可以看到 `taos-explorer` 服务。如果按照 GitHub 里步骤自己编译 TDengine 源代码，生成的安装包不会带 taosExplorer。 ## 配置 From 0ae67279fa10f7cd7e9562f566eefa983b3aeb6b Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Tue, 8 Oct 2024 08:30:29 +0800 Subject: [PATCH 13/61] Update 07-explorer.md --- docs/zh/14-reference/01-components/07-explorer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/01-components/07-explorer.md b/docs/zh/14-reference/01-components/07-explorer.md index 88269c502f..1539d3302e 100644 --- a/docs/zh/14-reference/01-components/07-explorer.md +++ b/docs/zh/14-reference/01-components/07-explorer.md @@ -4,7 +4,7 @@ sidebar_label: taosExplorer toc_max_heading_level: 4 --- -taosExplorer 是一个为用户提供 TDengine 实例的可视化管理交互工具的 web 服务, 没有开源，但随安装包免费提供。本节主要讲述其安装和部署。它的各项功能都是基于简单易上手的图形界面，可以直接尝试，如果有需要也可以考高级功能和运维指南中的相关内容。为了确保访问 taosExplorer 的最佳体验，请使用 Chrome 79 及以上版本，或 Edge 79 及以上版本。 +taosExplorer 是一个为用户提供 TDengine 实例的可视化管理交互工具的 web 服务，虽然它没有开源，但随开源版安装包免费提供。本节主要讲述其安装和部署。它的各项功能都是基于简单易上手的图形界面，可以直接尝试，如果有需要也可以参考高级功能和运维指南中的相关内容。为了确保访问 taosExplorer 的最佳体验，请使用 Chrome 79 及以上版本，或 Edge 79 及以上版本。 ## 安装 From f0680f5872ffe4a81959f8c39b1d3f182b7c04c1 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Tue, 8 Oct 2024 08:31:27 +0800 Subject: [PATCH 14/61] Update 07-explorer.md --- docs/zh/14-reference/01-components/07-explorer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/01-components/07-explorer.md b/docs/zh/14-reference/01-components/07-explorer.md index 1539d3302e..e13d3a2060 100644 --- a/docs/zh/14-reference/01-components/07-explorer.md +++ b/docs/zh/14-reference/01-components/07-explorer.md @@ -8,7 +8,7 @@ taosExplorer 是一个为用户提供 TDengine 实例的可视化管理交互工 ## 安装 -taosEexplorer 无需单独安装，从 TDengine 3.3.0.0 版本开始，它随着 TDengine 安装包一起发布，安装完成后，就可以看到 `taos-explorer` 服务。如果按照 GitHub 里步骤自己编译 TDengine 源代码，生成的安装包不会带 taosExplorer。 +taosEexplorer 无需单独安装，从 TDengine 3.3.0.0 版本开始，它随着 TDengine 安装包一起发布，安装完成后，就可以看到 `taos-explorer` 服务。如果按照 GitHub 里步骤自己编译 TDengine 源代码生成的安装包不包含 taosExplorer。 ## 配置 From 0d83a14d7ef020b5d04578a4ce2796aab7ac4d8a Mon Sep 17 00:00:00 2001 From: sheyanjie-qq <249478495@qq.com> Date: Tue, 8 Oct 2024 14:01:07 +0800 Subject: [PATCH 15/61] fix taos/taosws dll path issue --- docs/en/14-reference/05-connectors/10-cpp.mdx | 2 +- docs/zh/14-reference/05-connector/10-cpp.mdx | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/14-reference/05-connectors/10-cpp.mdx b/docs/en/14-reference/05-connectors/10-cpp.mdx index 6a570b2490..ca32660ac7 100644 --- a/docs/en/14-reference/05-connectors/10-cpp.mdx +++ b/docs/en/14-reference/05-connectors/10-cpp.mdx @@ -19,7 +19,7 @@ After TDengine server or client installation, `taos.h` is located at The dynamic libraries for the TDengine client driver are located in. - Linux: `/usr/local/taos/driver/libtaos.so` -- Windows: `C:\TDengine\taos.dll` +- Windows: `C:\TDengine\driver\taos.dll` - macOS: `/usr/local/lib/libtaos.dylib` ## Supported platforms diff --git a/docs/zh/14-reference/05-connector/10-cpp.mdx b/docs/zh/14-reference/05-connector/10-cpp.mdx index 0df6ed924c..c618601fb9 100644 --- a/docs/zh/14-reference/05-connector/10-cpp.mdx +++ b/docs/zh/14-reference/05-connector/10-cpp.mdx @@ -27,7 +27,7 @@ TDengine 服务端或客户端安装后，`taosws.h` 位于： TDengine 客户端驱动的动态库位于： - Linux: `/usr/local/taos/driver/libtaosws.so` -- Windows: `C:\TDengine\taosws.dll` +- Windows: `C:\TDengine\driver\taosws.dll` - macOS: `/usr/local/lib/libtaosws.dylib` ### 支持的平台 @@ -626,7 +626,7 @@ TDengine 服务端或客户端安装后，`taos.h` 位于： TDengine 客户端驱动的动态库位于： - Linux: `/usr/local/taos/driver/libtaos.so` -- Windows: `C:\TDengine\taos.dll` +- Windows: `C:\TDengine\driver\taos.dll` - macOS: `/usr/local/lib/libtaos.dylib` ### 支持的平台 From 4d482d8e4e81bfd64b28eca4909412d4e1c71081 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 9 Oct 2024 11:22:28 +0800 Subject: [PATCH 16/61] fix double free --- source/dnode/mnode/impl/src/mndIndex.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index 0b3a0998f0..718c34e85a 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -157,7 +157,7 @@ static void *mndBuildDropIdxReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStbOb pHead->contLen = htonl(len); pHead->vgId = htonl(pVgroup->vgId); - void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); int32_t ret = 0; if ((ret = tSerializeSDropIdxReq(pBuf, len - sizeof(SMsgHead), &req)) < 0) { terrno = ret; @@ -662,6 +662,8 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb pNew->pTags = NULL; pNew->pColumns = NULL; + pNew->pCmpr = NULL; + pNew->pTags = NULL; pNew->updateTime = taosGetTimestampMs(); pNew->lock = 0; From abbb06818341d39d12655c4f6a45223d1a915b6f Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 10 Oct 2024 13:59:45 +0800 Subject: [PATCH 17/61] fix windows, mac installer issue --- .../tools/com.taosdata.taos-explorer.plist | 33 +++++++++++++++++++ packaging/tools/remove.sh | 26 ++++++++++----- packaging/tools/tdengine.iss | 4 +-- 3 files changed, 53 insertions(+), 10 deletions(-) create mode 100644 packaging/tools/com.taosdata.taos-explorer.plist diff --git a/packaging/tools/com.taosdata.taos-explorer.plist b/packaging/tools/com.taosdata.taos-explorer.plist new file mode 100644 index 0000000000..2edb5552ad --- /dev/null +++ b/packaging/tools/com.taosdata.taos-explorer.plist @@ -0,0 +1,33 @@ + + + + + Label + com.tdengine.taos-explorer + ProgramArguments + + /usr/local/bin/taos-explorer + + ProcessType + Interactive + Disabled + + RunAtLoad + + LaunchOnlyOnce + + SessionCreate + + ExitTimeOut + 600 + KeepAlive + + SuccessfulExit + + AfterInitialDemand + + + Program + /usr/local/bin/taos-explorer + + \ No newline at end of file diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 58a17e2a50..c3f459ca9c 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -206,10 +206,17 @@ function clean_log() { } function clean_service_on_launchctl() { - ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : - ${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : - ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || : - ${csudo}rm /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || : + ${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist || : + ${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${PREFIX}adapter.plist || : + ${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${PREFIX}keeper.plist || : + ${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${PREFIX}-explorer.plist || : + + ${csudo}launchctl remove com.tdengine.taosd || : + ${csudo}launchctl remove com.tdengine.${PREFIX}adapter || : + ${csudo}launchctl remove com.tdengine.${PREFIX}keeper || : + ${csudo}launchctl remove com.tdengine.${PREFIX}-explorer || : + + ${csudo}rm /Library/LaunchDaemons/com.taosdata.* > /dev/null 2>&1 || : } function remove_data_and_config() { @@ -250,6 +257,12 @@ if [ -e ${install_main_dir}/uninstall_${PREFIX}x.sh ]; then fi fi + +if [ "$osType" = "Darwin" ]; then + clean_service_on_launchctl + ${csudo}rm -rf /Applications/TDengine.app +fi + remove_bin clean_header # Remove lib file @@ -282,10 +295,7 @@ elif echo $osinfo | grep -qwi "centos"; then # echo "this is centos system" ${csudo}rpm -e --noscripts tdengine >/dev/null 2>&1 || : fi -if [ "$osType" = "Darwin" ]; then - clean_service_on_launchctl - ${csudo}rm -rf /Applications/TDengine.app -fi + command -v systemctl >/dev/null 2>&1 && ${csudo}systemctl daemon-reload >/dev/null 2>&1 || true echo diff --git a/packaging/tools/tdengine.iss b/packaging/tools/tdengine.iss index 8085c55e3e..c3eb6f9f68 100644 --- a/packaging/tools/tdengine.iss +++ b/packaging/tools/tdengine.iss @@ -71,8 +71,8 @@ Source: {#MyAppSourceDir}\taosdump.exe; DestDir: "{app}"; DestName: "{#CusPrompt Filename: {sys}\sc.exe; Parameters: "create taosd start= DEMAND binPath= ""C:\\TDengine\\taosd.exe --win_service""" ; Flags: runhidden Filename: {sys}\sc.exe; Parameters: "create taosadapter start= DEMAND binPath= ""C:\\TDengine\\taosadapter.exe""" ; Flags: runhidden -Filename: "C:\Windows\System32\odbcconf.exe"; Parameters: "/S /F win_odbcinst.ini"; WorkingDir: "{app}\taos_odbc\x64"; Flags: runhidden; StatusMsg: "Configuring ODBC x64" -Filename: "C:\Windows\SysWOW64\odbcconf.exe"; Parameters: "/S /F win_odbcinst.ini"; WorkingDir: "{app}\taos_odbc\x86"; Flags: runhidden; StatusMsg: "Configuring ODBC x86" +Filename: "C:\Windows\System32\odbcconf.exe"; Parameters: "/S /F win_odbc_install.ini"; WorkingDir: "{app}\taos_odbc\x64"; Flags: runhidden; StatusMsg: "Configuring ODBC x64" +Filename: "C:\Windows\SysWOW64\odbcconf.exe"; Parameters: "/S /F win_odbc_install.ini"; WorkingDir: "{app}\taos_odbc\x86"; Flags: runhidden; StatusMsg: "Configuring ODBC x86" [UninstallRun] RunOnceId: "stoptaosd"; Filename: {sys}\sc.exe; Parameters: "stop taosd" ; Flags: runhidden From 838fdb49d067f8e7accfec38a77d74584e01d4bd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 10 Oct 2024 14:21:37 +0800 Subject: [PATCH 18/61] enh: change tsdb snapshot strategy to accelerate the snapshot process --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index d0ea58c28a..e8740a0650 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -623,6 +623,7 @@ static int32_t tsdbSnapWriteFileSetOpenReader(STsdbSnapWriter* writer) { int32_t lino = 0; if (writer->ctx->fset) { +#if 0 // open data reader SDataFileReaderConfig dataFileReaderConfig = { .tsdb = writer->tsdb, @@ -650,6 +651,7 @@ static int32_t tsdbSnapWriteFileSetOpenReader(STsdbSnapWriter* writer) { code = tsdbDataFileReaderOpen(NULL, &dataFileReaderConfig, &writer->ctx->dataReader); TSDB_CHECK_CODE(code, lino, _exit); +#endif // open stt reader array SSttLvl* lvl; @@ -791,6 +793,15 @@ static int32_t tsdbSnapWriteFileSetOpenWriter(STsdbSnapWriter* writer) { .did = writer->ctx->did, .level = 0, }; + // merge stt files to either data or a new stt file + if (writer->ctx->fset) { + for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (writer->ctx->fset->farr[ftype] != NULL) { + config.files[ftype].exist = true; + config.files[ftype].file = writer->ctx->fset->farr[ftype]->f[0]; + } + } + } code = tsdbFSetWriterOpen(&config, &writer->ctx->fsetWriter); TSDB_CHECK_CODE(code, lino, _exit); @@ -842,6 +853,8 @@ static int32_t tsdbSnapWriteFileSetBegin(STsdbSnapWriter* writer, int32_t fid) { _exit: if (code) { TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } else { + tsdbInfo("vgId:%d %s succeeded, fid:%d", TD_VID(writer->tsdb->pVnode), __func__, fid); } return code; } @@ -922,6 +935,8 @@ static int32_t tsdbSnapWriteFileSetEnd(STsdbSnapWriter* writer) { _exit: if (code) { TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } else { + tsdbInfo("vgId:%d %s succeeded, fid:%d", TD_VID(writer->tsdb->pVnode), __func__, writer->ctx->fid); } return code; } @@ -1175,7 +1190,7 @@ _exit: if (code) { TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); } else { - tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); + tsdbInfo("vgId:%d %s done, rollback:%d", TD_VID(tsdb->pVnode), __func__, rollback); } return code; } From 1f635c9ed9f7c5f51084cc67525308109db77420 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Thu, 10 Oct 2024 15:21:34 +0800 Subject: [PATCH 19/61] doc: fix broken links --- docs/zh/04-get-started/05-cloud.md | 2 +- docs/zh/08-operation/03-deployment.md | 4 ++-- docs/zh/08-operation/12-multi.md | 2 +- docs/zh/08-operation/18-dual.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/zh/04-get-started/05-cloud.md b/docs/zh/04-get-started/05-cloud.md index bd76add527..c8bd0f5dac 100644 --- a/docs/zh/04-get-started/05-cloud.md +++ b/docs/zh/04-get-started/05-cloud.md @@ -15,7 +15,7 @@ TDengine Cloud 大幅减轻了用户在部署、运维等方面的人力负担 要在 TDengine Cloud 注册新用户，请遵循以下简易步骤完成注册流程： -1. 打开浏览器，访问 TDengine Cloud 的首页：https://cloud.taosdata.com，在右边的“注册”部分，填入自己的姓名以及企业邮箱地址，点击“获取验证码”按钮。 +1. 打开浏览器，访问 TDengine Cloud[https://cloud.taosdata.com]，在右边的“注册”部分，填入自己的姓名以及企业邮箱地址，点击“获取验证码”按钮。 2. 检查企业邮箱，找到主题为“你的 TDengine Cloud 注册账户验证码”的邮件。从邮件内容中复制 6 位验证码，并将其粘贴到注册页面上的“验证码”输入框中。接着，点击“注册 TDengine Cloud”按钮，进入客户信息补全页面。 diff --git a/docs/zh/08-operation/03-deployment.md b/docs/zh/08-operation/03-deployment.md index 83b2c91843..2e0c2a7989 100644 --- a/docs/zh/08-operation/03-deployment.md +++ b/docs/zh/08-operation/03-deployment.md @@ -206,11 +206,11 @@ http { ### 部署 taosX -如果想使用 TDengine 的数据接入能力，需要部署 taosX 服务，关于它的详细说明和部署请参考[taosX 参考手册](../../reference/components/taosx)。 +如果想使用 TDengine 的数据接入能力，需要部署 taosX 服务，关于它的详细说明和部署请参考企业版参考手册。 ### 部署 taosX-Agent -有些数据源如 Pi, OPC 等，因为网络条件和数据源访问的限制，taosX 无法直接访问数据源，这种情况下需要部署一个代理服务 taosX-Agent，关于它的详细说明和部署请参考[taosX-Agent 参考手册](../../reference/components/taosx-agent)。 +有些数据源如 Pi, OPC 等，因为网络条件和数据源访问的限制，taosX 无法直接访问数据源，这种情况下需要部署一个代理服务 taosX-Agent，关于它的详细说明和部署请参考企业版参考手册。 ### 部署 taos-Explorer diff --git a/docs/zh/08-operation/12-multi.md b/docs/zh/08-operation/12-multi.md index 8f11ee4326..a5608ad5fa 100644 --- a/docs/zh/08-operation/12-multi.md +++ b/docs/zh/08-operation/12-multi.md @@ -70,7 +70,7 @@ dataDir /mnt/data6 2 0 |参数名称 | 参数含义 | |:-------------|:-----------------------------------------------| -|s3EndPoint | 用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 的保持一致，否则无法访问。例如：http://cos.ap-beijing.myqcloud.com | +|s3EndPoint | 用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 的保持一致，否则无法访问。 | |s3AccessKey |冒号分隔的用户 SecretId:SecretKey。例如：AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E | |s3BucketName | 存储桶名称，减号后面是用户注册 COS 服务的 AppId。其中 AppId 是 COS 特有，AWS 和阿里云都没有，配置时需要作为 bucket name 的一部分，使用减号分隔。参数值均为字符串类型，但不需要引号。例如：test0711-1309024725 | |s3UploadDelaySec | data 文件持续多长时间不再变动后上传至 s3，单位：秒。最小值：1；最大值：2592000 (30天），默认值 60 秒 | diff --git a/docs/zh/08-operation/18-dual.md b/docs/zh/08-operation/18-dual.md index 354e715602..c7871a8e1e 100644 --- a/docs/zh/08-operation/18-dual.md +++ b/docs/zh/08-operation/18-dual.md @@ -83,7 +83,7 @@ taosx replica start ```shell taosx replica start -f td1:6030 -t td2:6030 ``` -该示例命令会自动创建除 information_schema、performance_schema、log、audit 库之外的同步任务。可以使用 http://td2:6041 指定该 endpoint 使用 websocket 接口（默认是原生接口）。也可以指定数据库同步：taosx replica start -f td1:6030 -t td2:6030 db1 仅创建指定的数据库同步任务。 +该示例命令会自动创建除 information_schema、performance_schema、log、audit 库之外的同步任务。可以使用 `http://td2:6041` 指定该 endpoint 使用 websocket 接口（默认是原生接口）。也可以指定数据库同步：taosx replica start -f td1:6030 -t td2:6030 db1 仅创建指定的数据库同步任务。 2. 方法二 From c7dda0ca3a80959c60573e1f8cfcec029ab623d3 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 10 Oct 2024 07:24:33 +0000 Subject: [PATCH 20/61] fix/TS-5507-config-generate-key --- source/common/src/tglobal.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 2aef97ed1b..9a06acacb1 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1724,6 +1724,7 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char * TAOS_CHECK_RETURN(cfgInit(&pCfg)); TAOS_CHECK_GOTO(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE), NULL, _exit); + TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "debugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER), NULL, _exit); TAOS_CHECK_GOTO(cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) ,NULL, _exit); if ((code = taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) { From 567984130cb3a3b7e62cfc21fddc64700986f1f6 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:30:59 +0800 Subject: [PATCH 21/61] Update 05-cloud.md --- docs/zh/04-get-started/05-cloud.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/04-get-started/05-cloud.md b/docs/zh/04-get-started/05-cloud.md index c8bd0f5dac..1bca09ee91 100644 --- a/docs/zh/04-get-started/05-cloud.md +++ b/docs/zh/04-get-started/05-cloud.md @@ -15,7 +15,7 @@ TDengine Cloud 大幅减轻了用户在部署、运维等方面的人力负担 要在 TDengine Cloud 注册新用户，请遵循以下简易步骤完成注册流程： -1. 打开浏览器，访问 TDengine Cloud[https://cloud.taosdata.com]，在右边的“注册”部分，填入自己的姓名以及企业邮箱地址，点击“获取验证码”按钮。 +1. 打开浏览器，访问 [TDengine Cloud](https://cloud.taosdata.com)，在右边的“注册”部分，填入自己的姓名以及企业邮箱地址，点击“获取验证码”按钮。 2. 检查企业邮箱，找到主题为“你的 TDengine Cloud 注册账户验证码”的邮件。从邮件内容中复制 6 位验证码，并将其粘贴到注册页面上的“验证码”输入框中。接着，点击“注册 TDengine Cloud”按钮，进入客户信息补全页面。 @@ -32,4 +32,4 @@ TDengine Cloud 大幅减轻了用户在部署、运维等方面的人力负担 3. 第 3 步，创建实例。在此步骤中，你需要填写实例的区域、名称、是否选择高可用选项以及计费方案等必填信息。确认无误后，点击“创建”按钮。大约等待 1min，新的TDengine 实例便会创建完成。随后，你可以在控制台中对该实例进行各种操作，如查询数据、创建订阅、创建流等。 -TDengine Cloud 提供多种级别的计费方案，包括入门版、基础版、标准版、专业版和旗舰版，以满足不同客户的需求。如果你觉得现有计费方案无法满足自己的特定需求，请联系 TDengine Cloud 的客户支持团队，他们将为你量身定制计费方案。注册后，你将获得一定的免费额度，以便体验服务 \ No newline at end of file +TDengine Cloud 提供多种级别的计费方案，包括入门版、基础版、标准版、专业版和旗舰版，以满足不同客户的需求。如果你觉得现有计费方案无法满足自己的特定需求，请联系 TDengine Cloud 的客户支持团队，他们将为你量身定制计费方案。注册后，你将获得一定的免费额度，以便体验服务 From ceb4e4495a44bc4b88b2773260d5952401c76c00 Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Fri, 11 Oct 2024 09:07:12 +0800 Subject: [PATCH 22/61] docs:[TD-32510] Modify the description of function WEEKDAY. --- docs/zh/14-reference/03-taos-sql/10-function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/03-taos-sql/10-function.md b/docs/zh/14-reference/03-taos-sql/10-function.md index ac69f64f08..8c882b3237 100644 --- a/docs/zh/14-reference/03-taos-sql/10-function.md +++ b/docs/zh/14-reference/03-taos-sql/10-function.md @@ -1458,7 +1458,7 @@ WEEKDAY(expr) **适用于**: 表和超级表。 **使用说明**： -- 返回值 0 代表周日，1 代表周一 ... 6 代表周六。 +- 返回值 0 代表周一，1 代表周二 ... 6 代表周日。 - 若 `expr` 为 NULL，返回 NULL。 - 输入时间戳的精度由所查询表的精度确定, 若未指定表, 则精度为毫秒. From 6c9a91b26ba529649f454c0f4a9eab853f992df2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 11 Oct 2024 09:17:43 +0800 Subject: [PATCH 23/61] fix/TD-32505-main --- .../executor/src/streamtimewindowoperator.c | 85 ++++++++++--------- 1 file changed, 45 insertions(+), 40 deletions(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index e8322d6911..f5c85cb1da 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -49,7 +49,7 @@ #define STREAM_SESSION_OP_CHECKPOINT_NAME "StreamSessionOperator_Checkpoint" #define STREAM_STATE_OP_CHECKPOINT_NAME "StreamStateOperator_Checkpoint" -#define MAX_STREAM_HISTORY_RESULT 100000000 +#define MAX_STREAM_HISTORY_RESULT 100000000 typedef struct SStateWindowInfo { SResultWindowInfo winInfo; @@ -449,7 +449,7 @@ void destroyFlusedPos(void* pRes) { } void destroyFlusedppPos(void* ppRes) { - void *pRes = *(void **)ppRes; + void* pRes = *(void**)ppRes; destroyFlusedPos(pRes); } @@ -507,7 +507,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { } taosArrayDestroy(pInfo->pMidPullDatas); - if (pInfo->pState !=NULL && pInfo->pState->dump == 1) { + if (pInfo->pState != NULL && pInfo->pState->dump == 1) { taosMemoryFreeClear(pInfo->pState->pTdbState->pOwner); taosMemoryFreeClear(pInfo->pState->pTdbState); } @@ -548,7 +548,8 @@ void reloadFromDownStream(SOperatorInfo* downstream, SStreamIntervalOperatorInfo bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { return pInfo->primaryPkIndex != -1; } -int32_t initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo, struct SSteamOpBasicInfo* pBasic) { +int32_t initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo, + struct SSteamOpBasicInfo* pBasic) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore; @@ -1028,7 +1029,7 @@ static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pN } static int32_t doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId, - SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) { + SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info; @@ -1661,7 +1662,7 @@ static int32_t doStreamFinalIntervalAggNext(SOperatorInfo* pOperator, SSDataBloc pInfo->binfo.pRes->info.type = pBlock->info.type; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { - SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); + SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); QUERY_CHECK_NULL(delWins, code, lino, _end, terrno); SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL; code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap); @@ -1897,9 +1898,8 @@ _end: } } -int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo, int32_t numOfChild, - SReadHandle* pHandle, SOperatorInfo** pOptrInfo) { +int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, + int32_t numOfChild, SReadHandle* pHandle, SOperatorInfo** pOptrInfo) { QRY_PARAM_CHECK(pOptrInfo); int32_t code = TSDB_CODE_SUCCESS; @@ -1959,8 +1959,8 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex); - int32_t numOfCols = 0; - SExprInfo* pExprInfo = NULL; + int32_t numOfCols = 0; + SExprInfo* pExprInfo = NULL; code = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &pExprInfo, &numOfCols); QUERY_CHECK_CODE(code, lino, _error); @@ -2042,11 +2042,13 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN pOperator->info = pInfo; if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) { - pOperator->fpSet = createOperatorFpSet(NULL, doStreamMidIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo, - optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = + createOperatorFpSet(NULL, doStreamMidIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); } else { - pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo, - optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = + createOperatorFpSet(NULL, doStreamFinalIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); } setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || @@ -2220,10 +2222,10 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, in *(pSup->pState) = *pState; pSup->stateStore.streamStateSetNumber(pSup->pState, -1, tsIndex); int32_t funResSize = getMaxFunResSize(pExpSup, numOfOutput); - pSup->pState->pFileState = NULL; - code = pSup->stateStore.streamFileStateInit( - tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize, funResSize, sesionTs, pSup->pState, - pTwAggSup->deleteMark, taskIdStr, pHandle->checkpointId, STREAM_STATE_BUFF_SORT, &pSup->pState->pFileState); + pSup->pState->pFileState = NULL; + code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize, funResSize, + sesionTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr, + pHandle->checkpointId, STREAM_STATE_BUFF_SORT, &pSup->pState->pFileState); QUERY_CHECK_CODE(code, lino, _end); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); @@ -3309,8 +3311,8 @@ int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpe QUERY_CHECK_CONDITION((winCode == TSDB_CODE_SUCCESS), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); - code = - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo, + sizeof(SResultWindowInfo)); QUERY_CHECK_CODE(code, lino, _end); } @@ -3772,8 +3774,8 @@ _end: } } -int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pOptrInfo) { +int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, + SReadHandle* pHandle, SOperatorInfo** pOptrInfo) { QRY_PARAM_CHECK(pOptrInfo); SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; @@ -3802,12 +3804,12 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode } } SExprSupp* pExpSup = &pOperator->exprSupp; - + SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno); pInfo->binfo.pRes = pResBlock; - SExprInfo* pExprInfo = NULL; + SExprInfo* pExprInfo = NULL; code = createExprInfo(pSessionNode->window.pFuncs, NULL, &pExprInfo, &numOfCols); QUERY_CHECK_CODE(code, lino, _error); @@ -3886,8 +3888,9 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode QUERY_CHECK_CODE(code, lino, _error); } } - pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAggNext, NULL, destroyStreamSessionAggOperatorInfo, - optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = + createOperatorFpSet(optrDummyOpenFn, doStreamSessionAggNext, NULL, destroyStreamSessionAggOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState); if (downstream) { @@ -4102,8 +4105,8 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo, int32_t numOfChild, - SReadHandle* pHandle, SOperatorInfo** pOptrInfo) { + SExecTaskInfo* pTaskInfo, int32_t numOfChild, SReadHandle* pHandle, + SOperatorInfo** pOptrInfo) { QRY_PARAM_CHECK(pOptrInfo); int32_t code = TSDB_CODE_SUCCESS; @@ -4111,7 +4114,7 @@ int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhys SOperatorInfo* pOperator = NULL; code = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, &pOperator); if (pOperator == NULL || code != 0) { - downstream = NULL; + downstream = NULL; QUERY_CHECK_CODE(code, lino, _error); } @@ -4618,8 +4621,8 @@ int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera QUERY_CHECK_CODE(code, lino, _end); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); - code = - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo, + sizeof(SResultWindowInfo)); QUERY_CHECK_CODE(code, lino, _end); } @@ -4846,7 +4849,7 @@ void streamStateReleaseState(SOperatorInfo* pOperator) { int32_t resSize = winSize + sizeof(TSKEY); char* pBuff = taosMemoryCalloc(1, resSize); if (!pBuff) { - return ; + return; } memcpy(pBuff, pInfo->historyWins->pData, winSize); memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY)); @@ -4944,7 +4947,7 @@ void streamStateReloadState(SOperatorInfo* pOperator) { QUERY_CHECK_CODE(code, lino, _end); } } - taosMemoryFree(pBuf); + taosMemoryFreeClear(pBuf); SOperatorInfo* downstream = pOperator->pDownstream[0]; if (downstream->fpSet.reloadStreamStateFn) { @@ -4953,6 +4956,7 @@ void streamStateReloadState(SOperatorInfo* pOperator) { reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); _end: + taosMemoryFreeClear(pBuf); if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo)); } @@ -5001,9 +5005,9 @@ int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno); pInfo->binfo.pRes = pResBlock; - SExprSupp* pExpSup = &pOperator->exprSupp; - int32_t numOfCols = 0; - SExprInfo* pExprInfo = NULL; + SExprSupp* pExpSup = &pOperator->exprSupp; + int32_t numOfCols = 0; + SExprInfo* pExprInfo = NULL; code = createExprInfo(pStateNode->window.pFuncs, NULL, &pExprInfo, &numOfCols); QUERY_CHECK_CODE(code, lino, _error); @@ -5335,7 +5339,7 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex); - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; SExprInfo* pExprInfo = NULL; code = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &pExprInfo, &numOfCols); QUERY_CHECK_CODE(code, lino, _error); @@ -5383,7 +5387,8 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pInfo->pState->pFileState = NULL; code = pTaskInfo->storageAPI.stateStore.streamFileStateInit( tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState, - pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH, &pInfo->pState->pFileState); + pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH, + &pInfo->pState->pFileState); QUERY_CHECK_CODE(code, lino, _error); pInfo->pOperator = pOperator; @@ -5397,7 +5402,7 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pInfo->recvGetAll = false; code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes); - QUERY_CHECK_CODE(code, lino, _error); + QUERY_CHECK_CODE(code, lino, _error); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pDeletedMap = tSimpleHashInit(4096, hashFn); From ae0aa0b8212a8f381a08ee36aefe2c2fd6e92856 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 11 Oct 2024 09:31:35 +0800 Subject: [PATCH 24/61] fix invalid free or mem leak --- source/dnode/vnode/src/meta/metaTable.c | 7 ++++++- source/dnode/vnode/src/tsdb/tsdbFSetRW.c | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 8814e87140..08ee422126 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -3133,7 +3133,12 @@ static void colCompressDebug(SHashObj *pColCmprObj) { int32_t metaGetColCmpr(SMeta *pMeta, tb_uid_t uid, SHashObj **ppColCmprObj) { int rc = 0; - SHashObj *pColCmprObj = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); + SHashObj *pColCmprObj = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); + if (pColCmprObj == NULL) { + pColCmprObj = NULL; + return TSDB_CODE_OUT_OF_MEMORY; + } + void *pData = NULL; int nData = 0; SMetaEntry e = {0}; diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c index ea404142fe..60e42bd2b8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c @@ -46,8 +46,11 @@ static int32_t tsdbFSetWriteTableDataBegin(SFSetWriter *writer, const TABLEID *t code = tsdbUpdateSkmTb(writer->config->tsdb, writer->ctx->tbid, writer->skmTb); TSDB_CHECK_CODE(code, lino, _exit); + if (writer->pColCmprObj != NULL) { + taosHashCleanup(writer->pColCmprObj); + writer->pColCmprObj = NULL; + } code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, tbid->suid ? tbid->suid : tbid->uid, &writer->pColCmprObj); - // TODO: TSDB_CHECK_CODE(code, lino, _exit); writer->blockDataIdx = 0; for (int32_t i = 0; i < ARRAY_SIZE(writer->blockData); i++) { @@ -127,6 +130,8 @@ _exit: TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); } taosHashCleanup(writer->pColCmprObj); + writer->pColCmprObj = NULL; + return code; } From 6c1b1610a63a17e042e89188a62ee58676e4e363 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 11 Oct 2024 13:31:37 +0800 Subject: [PATCH 25/61] docs: supplement description of show commands --- docs/zh/14-reference/03-taos-sql/24-show.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/zh/14-reference/03-taos-sql/24-show.md b/docs/zh/14-reference/03-taos-sql/24-show.md index 09333dd0b7..71eb541361 100644 --- a/docs/zh/14-reference/03-taos-sql/24-show.md +++ b/docs/zh/14-reference/03-taos-sql/24-show.md @@ -30,6 +30,16 @@ SHOW CLUSTER ALIVE; 查询当前集群的状态是否可用，返回值： 0：不可用 1：完全可用 2：部分可用（集群中部分节点下线，但其它节点仍可以正常使用） +## SHOW CLUSTER MACHINES + +```sql +SHOW CLUSTER MACHINES; +``` + +显示集群的机器码等信息。 + +注：企业版独有 + ## SHOW CONNECTIONS ```sql @@ -99,6 +109,7 @@ SHOW FUNCTIONS; ```sql SHOW LICENCES; SHOW GRANTS; +SHOW GRANTS FULL; // 从 TDengine 3.2.3.0 版本开始支持 ``` 显示企业版许可授权的信息。 From 7332a596b6f07fcdaec38fba07e2956b19565552 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 11 Oct 2024 13:45:44 +0800 Subject: [PATCH 26/61] docs: supplement description of show commands --- docs/zh/14-reference/03-taos-sql/24-show.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/03-taos-sql/24-show.md b/docs/zh/14-reference/03-taos-sql/24-show.md index 71eb541361..51e9739def 100644 --- a/docs/zh/14-reference/03-taos-sql/24-show.md +++ b/docs/zh/14-reference/03-taos-sql/24-show.md @@ -33,7 +33,7 @@ SHOW CLUSTER ALIVE; ## SHOW CLUSTER MACHINES ```sql -SHOW CLUSTER MACHINES; +SHOW CLUSTER MACHINES; // 从 TDengine 3.2.3.0 版本开始支持 ``` 显示集群的机器码等信息。 From 3d22f879b0e5fcf629411f9c6a93c780182338be Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 11 Oct 2024 13:47:50 +0800 Subject: [PATCH 27/61] docs: supplement description of show commands --- docs/zh/14-reference/03-taos-sql/24-show.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/03-taos-sql/24-show.md b/docs/zh/14-reference/03-taos-sql/24-show.md index 51e9739def..110c9cee6e 100644 --- a/docs/zh/14-reference/03-taos-sql/24-show.md +++ b/docs/zh/14-reference/03-taos-sql/24-show.md @@ -33,7 +33,7 @@ SHOW CLUSTER ALIVE; ## SHOW CLUSTER MACHINES ```sql -SHOW CLUSTER MACHINES; // 从 TDengine 3.2.3.0 版本开始支持 +SHOW CLUSTER MACHINES; // 从 TDengine 3.2.3.0 版本开始支持 ``` 显示集群的机器码等信息。 From f94239cbb183ba15727fc0b5cd4e2fd4e4f2cbff Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 11 Oct 2024 13:59:57 +0800 Subject: [PATCH 28/61] add 3.3.3.0 release notes --- docs/zh/28-releases/03-notes/3.3.3.0.md | 112 ++++++++++++++++++++++++ docs/zh/28-releases/03-notes/index.md | 2 +- 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 docs/zh/28-releases/03-notes/3.3.3.0.md diff --git a/docs/zh/28-releases/03-notes/3.3.3.0.md b/docs/zh/28-releases/03-notes/3.3.3.0.md new file mode 100644 index 0000000000..b17933037f --- /dev/null +++ b/docs/zh/28-releases/03-notes/3.3.3.0.md @@ -0,0 +1,112 @@ +--- +title: 3.3.3.0 版本说明 +sidebar_label: 3.3.3.0 +description: 3.3.3.0 版本说明 +--- +### 新特性/优化 +1. 增加函数支持：pi、truncate/tunc、exp、ln、mod、rand、sign、degress、radians、char、ascii、position、trim、replace、repeat、substring、substr、substring_index、week、weekday、weekofyear、dayofweek、stddev_pop、var_pop +2. 多级存储支持微软对象存储 Azure Blob [企业版] +3. 支持 MongoDB 数据源 [企业版] +4. TDengine支持macOS企业版客户端 [企业版] +5. taosX日志默认不写入syslog [企业版] +6. 服务端记录所有慢查询信息到log库 +7. show cluster machines 查询结果中添加服务端版本号 +8. 删除保留关键字LEVEL/ENCODE/COMPRESS, 可以作为列名/表名/数据库名等使用 +9. 禁止动态修改临时目录 +10. round 函数：支持四舍五入的精度 +11. timediff 函数：返回时间戳 expr1 - expr2 的结果，支持负数结果，并近似到时间单位 time_unit 指定的精度 +12. max/min 函数：支持字符串作为输入参数，当输入参数为字符串类型时，返回最大字符串值 +13. 提升了 taosX 数据同步的性能 [企业版] +14. 统一慢查询和普通日志的保存策略 +15. 优化订阅功能中offset文件过大且加载过慢的问题 +16. 升级 JDBC driver 至 3.2.5 +17. Kafka connector 提交改为同步方式 +18. 新增参数：表查询不存在时是否报异常 +19. 扩大 tsma 的 interval 的最大窗口到 1 年 +20. 支持从 CSV 文件批量建表 +### 新特性/优化(企业版) +1. 对指定db进行balance vgroup leader +2. 多级存储新增配置项disable_create_new_file +3. 多级存储跨级迁移数据增加限速设置 +4. IP白名单启停支持热更新 +5. 普通用户取消建库权限 +6. 数据库加密改进密钥配置 +7. TDengine 2.0/3.0数据压缩的支持 +8. Oracle数据源支持 +9. 支持Microsoft SQL Server数据源 +10. OPC类型任务可动态获取新增点位 +11. PI backfill支持断点续传功能 +12. PI backfill类型的任务支持 Transformer +13. PI数据接入性能优化 +14. taos-explorer支持GEOMETRY/VARBINARY数据类型 +15. taos-explorer支持用户及权限信息的导入导出 +16. PI数据源支持新增数据点位/数据元素属性同步到TDengine +17. taosX写入端支持原生连接 +18. Kafka支持GSSAPI +19. MQTT类型任务可从数据源拉取示例数据 +20. 支持Object数组类型的数据 +21. 支持通过自定义脚本解析数据 +22. 支持通过插件的形式对数据动态筛选 +### 修复问题 +1. 服务器重启后 mqtt 写入任务未自动重新启动 [企业版] +2. 修复windows上agent和PI connector连接错误导致任务终止的问题 [企业版] +3. 重启taosadapter后,mqtt 任务异常，无法同步数据 [企业版] +4. Explorer 上 mqtt 写入任务状态不正确 [企业版] +5. 重启 taosx 后，运行中的任务没有被重新调度 [企业版] +6. 对于 taosx 的 MongoDB DataIn 任务，优化了日志的输出信息 [企业版] +7. 修复在 local.toml 文件为空时备份失败的问题 [企业版] +8. taosx replica 任务在网络断开恢复后数据同步异常的问题 [企业版] +9. 修复3.1版本同步到3.3版本meta数据不同步问题 [企业版] +10. taosadapter 异常重启后, mqtt任务无法自动恢复的问题[企业版] +11. Kafka DataIn 任务状态正常，但任务停止消费数据 [企业版] +12. 修复 last + interval 查询导致 crash 的问题 +13. 在数据写入时热更stt_trigger导致taosd崩溃 +14. 修改root密码后taoskeeper反复重启 +15. 云服务中服务重启后流计算无法继续 +16. tsdb加载缓存导致taosd死锁 +17. 查询时fill值指定为0报错 +18. last_row 查询结果与预期不符 +19. 用户指定 information_schema 库登录时导致 taosd 崩溃 +20. group by 语句支持位置参数和别名语法 +21. 元数据克隆函数对象错误导致 crash +22. 缓存更新时数据填充的游标错误导致的 taosd 异常退出 +23. STDDEV 函数计算结果随机不正确的问题 +24. 多级存储以及加密场景下无法添加离线节点 +25. taos CLI 无法输入大于20字节长度的密码 +26. 拼接 sql 写入报错: int data overflow +27. 大量查询并发场景下元数据的一致性 +28. 尝试解决手动点击停止按钮，任务无法停止的问题 +29. 解决了列或者标签中包含逗号或引号时导出 CSV 文件错误的问题 +30. 多线程并发的对同一张子表进行 describe 导致 crash +31. 数据浏览器的查询结果未按照配置的时区展示的问题 +32. 创建 websocket 连接时 taosadapter 内存泄漏的问题 +33. 社区版无法在 redhat 操作系统正常启动 +34. 无效断言在两个 stt 分别位于数据块时间轴分布空洞场景引发的 crash +35. S3 长时间拉取数据会失败的问题 +36. 使用结果集函数判断时间线的有序性 +37. 解决超级表投影查询慢的问题 +38. 修复: 多节点环境，事务返回码与事务状态不一致导致事务堆积 +39. 修复：在stt_trigger = 1时，delete数据之后重新出现的错误 +40. 节点恢复阶段taosd内存占用过高导致OOM +41. limit过小时的判断错误 +42. AVEVA historian 数据源连通性及获取示例数据功能 +43. taosd 服务无法正常停止的问题 +44. last 函数查询全空列导致 crash +45. 查询 cid 不在当前版本的 schema 时触发 assert 导致 coredump +46. 写入消息体过大时 OOM 的问题，增加 syncLogBufferMemoryAllowed 参数控制 +47. timezone 设置为 UTC 且数据包含“1970”时,查询报错 +48. 取消mnode节点的消息同步的等待超时 +49. 集群节点异常恢复后，集群授权状态失效 +50. 查询空指针问题 +51. 在restore vnode时，将vgroup暂时设置成单副本 +52. 可以使用SQL保留字作为列名创建超级表，也可以使用SQL保留字作为标签名 +53. 高并发查询期间内存持续增长的问题 +54. 流计算在checkpoint处理过程中，由于断言失败导致的crash +55. 修复了 3.3.2.8.0820 版本中可以查询到 information_schema 中不存在的表perf_queries问题 +56. 节点恢复阶段taosd内存占用过高导致OOM +57. 修复在schema变更情况下，通过taosx导入数据时，taosd出现crash的问题 +58. 可能的内存泄漏 +59. 客户端内存泄漏 +60. 开源用户修改stt_trigger值升级后无法修改其他数据库选项 +61. NOT IN (NULL) 查询结果不正确 +62. taos shell和taosBenchmark不能成功连接云服务实例 diff --git a/docs/zh/28-releases/03-notes/index.md b/docs/zh/28-releases/03-notes/index.md index eadb434f53..4f639212c2 100644 --- a/docs/zh/28-releases/03-notes/index.md +++ b/docs/zh/28-releases/03-notes/index.md @@ -3,5 +3,5 @@ title: 版本说明 sidebar_label: 版本说明 description: 各版本版本说明 --- - +[3.3.3.0](./3.3.3.0) [3.3.2.0](./3.3.2.0) From 481e5aa9440af121d3ae416a30c8a71b0c1f3a39 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 11 Oct 2024 14:36:58 +0800 Subject: [PATCH 29/61] update --- docs/zh/28-releases/03-notes/3.3.3.0.md | 23 ----------------------- docs/zh/28-releases/03-notes/index.md | 2 +- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/docs/zh/28-releases/03-notes/3.3.3.0.md b/docs/zh/28-releases/03-notes/3.3.3.0.md index b17933037f..405ca83d71 100644 --- a/docs/zh/28-releases/03-notes/3.3.3.0.md +++ b/docs/zh/28-releases/03-notes/3.3.3.0.md @@ -24,29 +24,6 @@ description: 3.3.3.0 版本说明 18. 新增参数：表查询不存在时是否报异常 19. 扩大 tsma 的 interval 的最大窗口到 1 年 20. 支持从 CSV 文件批量建表 -### 新特性/优化(企业版) -1. 对指定db进行balance vgroup leader -2. 多级存储新增配置项disable_create_new_file -3. 多级存储跨级迁移数据增加限速设置 -4. IP白名单启停支持热更新 -5. 普通用户取消建库权限 -6. 数据库加密改进密钥配置 -7. TDengine 2.0/3.0数据压缩的支持 -8. Oracle数据源支持 -9. 支持Microsoft SQL Server数据源 -10. OPC类型任务可动态获取新增点位 -11. PI backfill支持断点续传功能 -12. PI backfill类型的任务支持 Transformer -13. PI数据接入性能优化 -14. taos-explorer支持GEOMETRY/VARBINARY数据类型 -15. taos-explorer支持用户及权限信息的导入导出 -16. PI数据源支持新增数据点位/数据元素属性同步到TDengine -17. taosX写入端支持原生连接 -18. Kafka支持GSSAPI -19. MQTT类型任务可从数据源拉取示例数据 -20. 支持Object数组类型的数据 -21. 支持通过自定义脚本解析数据 -22. 支持通过插件的形式对数据动态筛选 ### 修复问题 1. 服务器重启后 mqtt 写入任务未自动重新启动 [企业版] 2. 修复windows上agent和PI connector连接错误导致任务终止的问题 [企业版] diff --git a/docs/zh/28-releases/03-notes/index.md b/docs/zh/28-releases/03-notes/index.md index 4f639212c2..132a72d9ca 100644 --- a/docs/zh/28-releases/03-notes/index.md +++ b/docs/zh/28-releases/03-notes/index.md @@ -3,5 +3,5 @@ title: 版本说明 sidebar_label: 版本说明 description: 各版本版本说明 --- -[3.3.3.0](./3.3.3.0) +[3.3.3.0](./3.3.3.0) [3.3.2.0](./3.3.2.0) From 30fdf4964496ff92f9d74d7ecd22861a5407a7d8 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 11 Oct 2024 16:43:35 +0800 Subject: [PATCH 30/61] update tag --- cmake/stub_CMakeLists.txt.in | 3 +-- cmake/taosadapter_CMakeLists.txt.in | 2 +- cmake/taostools_CMakeLists.txt.in | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cmake/stub_CMakeLists.txt.in b/cmake/stub_CMakeLists.txt.in index 6c54d33be7..8a5fda87ac 100644 --- a/cmake/stub_CMakeLists.txt.in +++ b/cmake/stub_CMakeLists.txt.in @@ -9,6 +9,5 @@ ExternalProject_Add(stub CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" - TEST_COMMAND "" - GIT_SHALLOW true + TEST_COMMAND "" ) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index ef6ed4af1d..13826a1a74 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 3.0 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 9a6a5329ae..9bbda8309f 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 3.0 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From b5453909895041a21b71f9c0a34337e0395cafac Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Fri, 11 Oct 2024 18:44:07 +0800 Subject: [PATCH 31/61] Update 3.3.3.0.md --- docs/zh/28-releases/03-notes/3.3.3.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/28-releases/03-notes/3.3.3.0.md b/docs/zh/28-releases/03-notes/3.3.3.0.md index 405ca83d71..bb7bc0f831 100644 --- a/docs/zh/28-releases/03-notes/3.3.3.0.md +++ b/docs/zh/28-releases/03-notes/3.3.3.0.md @@ -10,7 +10,7 @@ description: 3.3.3.0 版本说明 4. TDengine支持macOS企业版客户端 [企业版] 5. taosX日志默认不写入syslog [企业版] 6. 服务端记录所有慢查询信息到log库 -7. show cluster machines 查询结果中添加服务端版本号 +7. show cluster machines 查询结果中添加服务端版本号 [企业版] 8. 删除保留关键字LEVEL/ENCODE/COMPRESS, 可以作为列名/表名/数据库名等使用 9. 禁止动态修改临时目录 10. round 函数：支持四舍五入的精度 From 2e9eb4799fe4f672dbd12e5ca3d67877229ce424 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 11 Oct 2024 18:34:08 +0800 Subject: [PATCH 32/61] fix(tsdb/scan): ignore lcn file only --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index d3b783847c..0c9d9e56cf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -433,6 +433,21 @@ static int32_t tsdbFSCreateFileObjHash(STFileSystem *fs, STFileHash *hash) { if (fset->farr[i] != NULL) { code = tsdbFSAddEntryToFileObjHash(hash, fset->farr[i]->fname); TSDB_CHECK_CODE(code, lino, _exit); + + if (TSDB_FTYPE_DATA == i && fset->farr[i]->f->lcn > 0) { + STFileObj *fobj = fset->farr[i]; + int32_t lcn = fobj->f->lcn; + char lcn_name[TSDB_FILENAME_LEN]; + + snprintf(lcn_name, TSDB_FQDN_LEN, "%s", fobj->fname); + char *dot = strrchr(lcn_name, '.'); + if (dot) { + snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lcn_name), "%d.data", lcn); + + code = tsdbFSAddEntryToFileObjHash(hash, lcn_name); + TSDB_CHECK_CODE(code, lino, _exit); + } + } } } @@ -535,9 +550,7 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) { for (const STfsFile *file = NULL; (file = tfsReaddir(dir)) != NULL;) { if (taosIsDir(file->aname)) continue; - if (tsdbFSGetFileObjHashEntry(&fobjHash, file->aname) == NULL && - strncmp(file->aname + strlen(file->aname) - 3, ".cp", 3) && - strncmp(file->aname + strlen(file->aname) - 5, ".data", 5)) { + if (tsdbFSGetFileObjHashEntry(&fobjHash, file->aname) == NULL) { tsdbRemoveFile(file->aname); } } From cf28726049c694175215975d233eba10ff527483 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 11 Oct 2024 19:23:03 +0800 Subject: [PATCH 33/61] merge main.20240929 --- packaging/tools/TDengine.pkgproj | 14 +++++++++++++- packaging/tools/mac_install_summary.txt | 13 +++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 packaging/tools/mac_install_summary.txt diff --git a/packaging/tools/TDengine.pkgproj b/packaging/tools/TDengine.pkgproj index 7aaac62735..3171d80d71 100644 --- a/packaging/tools/TDengine.pkgproj +++ b/packaging/tools/TDengine.pkgproj @@ -663,7 +663,19 @@ SUMMARY LOCALIZATIONS - + + + LANGUAGE + English + VALUE + + PATH + /opt/taos/TDengine/packaging/tools/mac_install_summary.txt + PATH_TYPE + 0 + + + TITLE diff --git a/packaging/tools/mac_install_summary.txt b/packaging/tools/mac_install_summary.txt new file mode 100644 index 0000000000..d13f1c280b --- /dev/null +++ b/packaging/tools/mac_install_summary.txt @@ -0,0 +1,13 @@ +TDengine is installed successfully. Please open an Mac terminal and execute the commands below: + +To configure TDengine, sudo vi /etc/taos/taos.cfg +To configure taosadapter, sudo vi /etc/taos/taoadapter.toml +To configure taos-explorer, sudo vi /etc/taos/explorer.toml +To start service, sudo launchctl start com.tdengine.taosd +To start Taos Adapter, sudo launchctl start com.tdengine.taosadapter +To start Taos Explorer, sudo launchctl start com.tdengine.taos-explorer + +To start all the components, sudo start-all.sh +To access TDengine Commnd Line Interface, taos -h YourServerName +To access TDengine Graphic User Interface, http://YourServerName:6060 +To read the user manual, http://YourServerName:6060/docs-en From 8cb19bb77c7f7670cd1b8914fbdc403e3fe7fa8b Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Sat, 12 Oct 2024 10:52:52 +0800 Subject: [PATCH 34/61] docs: add intro for create tb by csv --- docs/zh/14-reference/03-taos-sql/03-table.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/zh/14-reference/03-taos-sql/03-table.md b/docs/zh/14-reference/03-taos-sql/03-table.md index 2f0ae7100a..cad9190bd9 100644 --- a/docs/zh/14-reference/03-taos-sql/03-table.md +++ b/docs/zh/14-reference/03-taos-sql/03-table.md @@ -79,6 +79,18 @@ CREATE TABLE [IF NOT EXISTS] tb_name1 USING stb_name TAGS (tag_value1, ...) [IF 批量建表方式要求数据表必须以超级表为模板。 在不超出 SQL 语句长度限制的前提下，单条语句中的建表数量建议控制在 1000 ～ 3000 之间，将会获得比较理想的建表速度。 +### 使用 CSV 批量创建子表 + +```sql +CREATE TABLE [IF NOT EXISTS] USING [db_name.]stb_name (field1_name [, field2_name] ....) FILE csv_file_path; +``` + +**参数说明** + +1. FILE 语法表示数据来自于 CSV 文件（英文逗号分隔、英文单引号括住每个值），CSV 文件无需表头。CSV 文件中应仅包含 table name 与 tag 值。如需插入数据，请参考数据写入章节。 +2. 为指定的 stb_name 创建子表，该超级表必须已经存在。 +3. field_name 列表顺序与 CSV 文件各列内容顺序一致。列表中不允许出现重复项，且必须包含 `tbname`，可包含零个或多个超级表中已定义的标签列。未包含在列表中的标签值将被设置为 NULL。 + ## 修改普通表 ```sql From 78dc28463738fc116e875eb79244750d8c59ca06 Mon Sep 17 00:00:00 2001 From: zhiyong Date: Mon, 14 Oct 2024 09:00:17 +0800 Subject: [PATCH 35/61] update data.precision --- .../10-third-party/01-collection/11-kafka.md | 52 +++++++++---------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/docs/zh/10-third-party/01-collection/11-kafka.md b/docs/zh/10-third-party/01-collection/11-kafka.md index 75adefbc50..5113519e50 100644 --- a/docs/zh/10-third-party/01-collection/11-kafka.md +++ b/docs/zh/10-third-party/01-collection/11-kafka.md @@ -347,14 +347,18 @@ curl -X DELETE http://localhost:8083/connectors/TDengineSourceConnector 以下配置项对 TDengine Sink Connector 和 TDengine Source Connector 均适用。 1. `name`: connector 名称。 -2. `connector.class`: connector 的完整类名， 如: com.taosdata.kafka.connect.sink.TDengineSinkConnector。 -3. `tasks.max`: 最大任务数, 默认 1。 -4. `topics`: 需要同步的 topic 列表， 多个用逗号分隔, 如 `topic1,topic2`。 -5. `connection.url`: TDengine JDBC 连接字符串， 如 `jdbc:TAOS://127.0.0.1:6030`。 -6. `connection.user`： TDengine 用户名， 默认 root。 -7. `connection.password` ：TDengine 用户密码， 默认 taosdata。 -8. `connection.attempts` ：最大尝试连接次数。默认 3。 -9. `connection.backoff.ms` ： 创建连接失败重试时间隔时间，单位为 ms。 默认 5000。 +1. `connector.class`: connector 的完整类名， 如: com.taosdata.kafka.connect.sink.TDengineSinkConnector。 +1. `tasks.max`: 最大任务数, 默认 1。 +1. `topics`: 需要同步的 topic 列表， 多个用逗号分隔, 如 `topic1,topic2`。 +1. `connection.url`: TDengine JDBC 连接字符串， 如 `jdbc:TAOS://127.0.0.1:6030`。 +1. `connection.user`： TDengine 用户名， 默认 root。 +1. `connection.password` ：TDengine 用户密码， 默认 taosdata。 +1. `connection.attempts` ：最大尝试连接次数。默认 3。 +1. `connection.backoff.ms` ： 创建连接失败重试时间隔时间，单位为 ms。 默认 5000。 +1. `data.precision`: 使用 InfluxDB 行协议格式时，时间戳的精度。可选值为： + 1. ms ： 表示毫秒 + 1. us ： 表示微秒 + 1. ns ： 表示纳秒 ### TDengine Sink Connector 特有的配置 @@ -367,30 +371,22 @@ curl -X DELETE http://localhost:8083/connectors/TDengineSourceConnector 1. line ：代表 InfluxDB 行协议格式 2. json : 代表 OpenTSDB JSON 格式 3. telnet ：代表 OpenTSDB Telnet 行协议格式 -7. `data.precision`: 使用 InfluxDB 行协议格式时，时间戳的精度。可选值为： - 1. ms ： 表示毫秒 - 2. us ： 表示微秒 - 3. ns ： 表示纳秒。 ### TDengine Source Connector 特有的配置 1. `connection.database`: 源数据库名称，无缺省值。 -2. `topic.prefix`： 数据导入 kafka 时使用的 topic 名称的前缀。默认为空字符串 ""。 -3. `timestamp.initial`: 数据同步起始时间。格式为'yyyy-MM-dd HH:mm:ss'，若未指定则从指定 DB 中最早的一条记录开始。 -4. `poll.interval.ms`: 检查是否有新建或删除的表的时间间隔，单位为 ms。默认为 1000。 -5. `fetch.max.rows` : 检索数据库时最大检索条数。 默认为 100。 -6. `query.interval.ms`: 从 TDengine 一次读取数据的时间跨度，需要根据表中的数据特征合理配置，避免一次查询的数据量过大或过小；在具体的环境中建议通过测试设置一个较优值，默认值为 0，即获取到当前最新时间的所有数据。 -7. `out.format` : 结果集输出格式。`line` 表示输出格式为 InfluxDB Line 协议格式，`json` 表示输出格式是 json。默认为 line。 -8. `data.precision`: 使用 InfluxDB 行协议格式时，时间戳的精度。可选值为： - 1. ms ： 表示毫秒， - 2. us ： 表示微秒 - 3. ns ： 表示纳秒。 -9. `topic.per.stable`: 如果设置为 true，表示一个超级表对应一个 Kafka topic，topic的命名规则 ``；如果设置为 false，则指定的 DB 中的所有数据进入一个 Kafka topic，topic 的命名规则为 `` -10. `topic.ignore.db`: topic 命名规则是否包含 database 名称，true 表示规则为 ``，false 表示规则为 ``，默认 false。此配置项在 `topic.per.stable` 设置为 false 时不生效。 -11. `topic.delimiter`: topic 名称分割符，默认为 `-`。 -12. `read.method`: 从 TDengine 读取数据方式，query 或是 subscription。默认为 subscription。 -13. `subscription.group.id`: 指定 TDengine 数据订阅的组 id，当 `read.method` 为 subscription 时，此项为必填项。 -14. `subscription.from`: 指定 TDengine 数据订阅起始位置，latest 或是 earliest。默认为 latest。 +1. `topic.prefix`： 数据导入 kafka 时使用的 topic 名称的前缀。默认为空字符串 ""。 +1. `timestamp.initial`: 数据同步起始时间。格式为'yyyy-MM-dd HH:mm:ss'，若未指定则从指定 DB 中最早的一条记录开始。 +1. `poll.interval.ms`: 检查是否有新建或删除的表的时间间隔，单位为 ms。默认为 1000。 +1. `fetch.max.rows` : 检索数据库时最大检索条数。 默认为 100。 +1. `query.interval.ms`: 从 TDengine 一次读取数据的时间跨度，需要根据表中的数据特征合理配置，避免一次查询的数据量过大或过小；在具体的环境中建议通过测试设置一个较优值，默认值为 0，即获取到当前最新时间的所有数据。 +1. `out.format` : 结果集输出格式。`line` 表示输出格式为 InfluxDB Line 协议格式，`json` 表示输出格式是 json。默认为 line。 +1. `topic.per.stable`: 如果设置为 true，表示一个超级表对应一个 Kafka topic，topic的命名规则 ``；如果设置为 false，则指定的 DB 中的所有数据进入一个 Kafka topic，topic 的命名规则为 `` +1. `topic.ignore.db`: topic 命名规则是否包含 database 名称，true 表示规则为 ``，false 表示规则为 ``，默认 false。此配置项在 `topic.per.stable` 设置为 false 时不生效。 +1. `topic.delimiter`: topic 名称分割符，默认为 `-`。 +1. `read.method`: 从 TDengine 读取数据方式，query 或是 subscription。默认为 subscription。 +1. `subscription.group.id`: 指定 TDengine 数据订阅的组 id，当 `read.method` 为 subscription 时，此项为必填项。 +1. `subscription.from`: 指定 TDengine 数据订阅起始位置，latest 或是 earliest。默认为 latest。 ## 其他说明 From df1060a68a83f7f238338047b3959d44ee645f80 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 14 Oct 2024 17:33:00 +0800 Subject: [PATCH 36/61] merge docker packaging updates --- packaging/docker/Dockerfile | 2 +- packaging/docker/bin/entrypoint.sh | 53 +++++++++++++++++++++++------- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index 99a10e0285..a67724d5a8 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:latest +FROM ubuntu:22.04 WORKDIR /root diff --git a/packaging/docker/bin/entrypoint.sh b/packaging/docker/bin/entrypoint.sh index a60254d7ef..02895f4755 100755 --- a/packaging/docker/bin/entrypoint.sh +++ b/packaging/docker/bin/entrypoint.sh @@ -10,6 +10,12 @@ fi DISABLE_ADAPTER=${TAOS_DISABLE_ADAPTER:-0} unset TAOS_DISABLE_ADAPTER +DISABLE_KEEPER=${TAOS_DISABLE_KEEPER:-0} +unset TAOS_DISABLE_KEEPER + +DISABLE_EXPLORER=${TAOS_DISABLE_EXPLORER:-0} +unset TAOS_DISABLE_EXPLORER + # to get mnodeEpSet from data dir DATA_DIR=$(taosd -C|grep -E 'dataDir.*(\S+)' -o |head -n1|sed 's/dataDir *//') DATA_DIR=${DATA_DIR:-/var/lib/taos} @@ -33,26 +39,18 @@ ulimit -c unlimited sysctl -w kernel.core_pattern=/corefile/core-$FQDN-%e-%p >/dev/null >&1 set -e -if [ "$DISABLE_ADAPTER" = "0" ]; then - which taosadapter >/dev/null && taosadapter & - # wait for 6041 port ready - for _ in $(seq 1 20); do - nc -z localhost 6041 && break - sleep 0.5 - done -fi + # if dnode has been created or has mnode ep set or the host is first ep or not for cluster, just start. if [ -f "$DATA_DIR/dnode/dnode.json" ] || [ -f "$DATA_DIR/dnode/mnodeEpSet.json" ] || [ "$TAOS_FQDN" = "$FIRST_EP_HOST" ]; then - $@ + $@ & # others will first wait the first ep ready. else if [ "$TAOS_FIRST_EP" = "" ]; then echo "run TDengine with single node." - $@ - exit $? + $@ & fi while true; do es=$(taos -h $FIRST_EP_HOST -P $FIRST_EP_PORT --check | grep "^[0-9]*:") @@ -64,5 +62,36 @@ else fi sleep 1s done - $@ + if ps aux | grep -v grep | grep taosd > dev/null; then + echo "TDengine is running" + else + $@ & + fi fi + +if [ "$DISABLE_ADAPTER" = "0" ]; then + which taosadapter >/dev/null && taosadapter & + # wait for 6041 port ready + for _ in $(seq 1 20); do + nc -z localhost 6041 && break + sleep 0.5 + done +fi + +if [ "$DISABLE_KEEPER" = "0" ]; then + sleep 3 + which taoskeeper >/dev/null && taoskeeper & + # wait for 6043 port ready + for _ in $(seq 1 20); do + nc -z localhost 6043 && break + sleep 0.5 + done +fi + + +which taos-explorer >/dev/null && taos-explorer +# wait for 6060 port ready +for _ in $(seq 1 20); do + nc -z localhost 6060 && break + sleep 0.5 +done From b21693063dd0accf61b8c59b7c44c346d9f3cd0a Mon Sep 17 00:00:00 2001 From: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Date: Mon, 14 Oct 2024 18:10:05 +0800 Subject: [PATCH 37/61] Update 10-taosbenchmark.md taosbenchmark datatype taosbenchmark datatype add varbinary and geometry --- docs/zh/14-reference/02-tools/10-taosbenchmark.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/zh/14-reference/02-tools/10-taosbenchmark.md b/docs/zh/14-reference/02-tools/10-taosbenchmark.md index 3f15d6b8e3..3c43c58915 100644 --- a/docs/zh/14-reference/02-tools/10-taosbenchmark.md +++ b/docs/zh/14-reference/02-tools/10-taosbenchmark.md @@ -477,7 +477,9 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) | 13 | BOOL | bool | 14 | NCHAR | nchar | 15 | VARCHAR | varchar -| 15 | JSON | json +| 16 | VARBINARY | varbinary +| 17 | GEOMETRY | geometry +| 18 | JSON | json 注意：taosBenchmark 配置文件中数据类型必须小写方可识别 From 991c5a08f73c2d38ad61db6aa12b1da3ff4d53b5 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 16 Oct 2024 11:06:59 +0800 Subject: [PATCH 38/61] fix:[TS-5528] insert error in sml --- source/client/inc/clientSml.h | 1 + source/client/src/clientSml.c | 8 +++++-- source/client/src/clientSmlLine.c | 9 ++++++++ tests/system-test/2-query/sml.py | 4 ++++ utils/test/c/sml_test.c | 35 +++++++++++++++++++++++++++++++ 5 files changed, 55 insertions(+), 2 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 209c376f30..a6aca2fddf 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -199,6 +199,7 @@ typedef struct { SArray *preLineTagKV; SArray *maxTagKVs; SArray *maxColKVs; + SArray *escapedStringList; SSmlLineInfo preLine; STableMeta *currSTableMeta; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index f3a22bff75..5a51021a3a 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -479,6 +479,7 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) } clearColValArraySml(info->currTableDataCtx->pValues); + taosArrayClearP(info->escapedStringList, taosMemoryFree); if (unlikely(ret != TSDB_CODE_SUCCESS)) { smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); return ret; @@ -1609,6 +1610,7 @@ void smlDestroyInfo(SSmlHandle *info) { taosArrayDestroy(info->valueJsonArray); taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv); + taosArrayDestroyP(info->escapedStringList, taosMemoryFree); if (!info->dataFormat) { for (int i = 0; i < info->lineNum; i++) { @@ -1668,8 +1670,9 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); info->valueJsonArray = taosArrayInit(8, POINTER_BYTES); info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); - - if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || info->preLineTagKV == NULL) { + info->escapedStringList = taosArrayInit(8, POINTER_BYTES); + if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || + info->preLineTagKV == NULL || info->escapedStringList == NULL) { uError("SML:0x%" PRIx64 " failed to allocate memory", info->id); code = terrno; goto FAILED; @@ -1950,6 +1953,7 @@ int32_t smlClearForRerun(SSmlHandle *info) { } } + taosArrayClearP(info->escapedStringList, taosMemoryFree); (void)memset(&info->preLine, 0, sizeof(SSmlLineInfo)); info->currSTableMeta = NULL; info->currTableDataCtx = NULL; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index e620ca9b0c..c1f3431698 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -451,6 +451,13 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (info->dataFormat) { bool isAligned = isSmlColAligned(info, cnt, &kv); + if (kv.type == TSDB_DATA_TYPE_BINARY && valueEscaped) { + if (taosArrayPush(info->escapedStringList, &kv.value) == NULL){ + freeSSmlKv(&kv); + return terrno; + } + kv.value = NULL; + } freeSSmlKv(&kv); if(!isAligned){ return TSDB_CODE_SUCCESS; @@ -459,10 +466,12 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (currElement->colArray == NULL) { currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); if (currElement->colArray == NULL) { + freeSSmlKv(&kv); return terrno; } } if (taosArrayPush(currElement->colArray, &kv) == NULL){ // reserve for timestamp + freeSSmlKv(&kv); return terrno; } } diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index e28f3b1edd..4a8760cd67 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -105,6 +105,10 @@ class TDTestCase: tdSql.query(f"select * from ts3724.`stb2.`") tdSql.checkRows(1) + tdSql.query(f"select * from ts5528.device_log_yuelan_cs1") + tdSql.checkRows(2) + tdSql.checkData(0, 1, '{"deviceId":"星宇公司-861701069493741","headers":{"_uid":"4e3599eacd62834995c77b38ad95f88d","creatorId":"1199596756811550720","deviceNmae":"861701069493741","productId":"yuelan","productName":"悦蓝cat1穿戴设备"},"messageType":"REPORT_PROPERTY","properties":{"lat":35.265527067449185,"lng":118.49713144245987,"location":"118.49713144245987,35.265527067449185"},"timestamp":1728719963230}') + tdSql.checkData(1, 1, '{"deviceId":"星宇公司-861701069065507","headers":{"_uid":"9045d6b78b4ffaf1e2d244e912ebbff8","creatorId":"1199596756811550720","deviceNmae":"861701069065507","productId":"yuelan","productName":"悦蓝cat1穿戴设备"},"messageType":"REPORT_PROPERTY","properties":{"lat":36.788241914043425,"lng":119.15042325460891,"location":"119.15042325460891,36.788241914043425"},"timestamp":1728719964105}') # tdSql.query(f"select * from td24559.stb order by _ts") # tdSql.checkRows(4) # tdSql.checkData(0, 2, "POINT (4.343000 89.342000)") diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index a3830d2518..55fa336221 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -2098,12 +2098,47 @@ int sml_td29373_Test() { return code; } +int sml_ts5528_test(){ + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "drop database if exists ts5528"); + taos_free_result(pRes); + + pRes = taos_query(taos, "create database if not exists ts5528"); + taos_free_result(pRes); + + // check column name duplication + char *sql[] = { + "device_log_yuelan_cs1,deviceId=861701069493741 content=\"{\\\"deviceId\\\":\\\"星宇公司-861701069493741\\\",\\\"headers\\\":{\\\"_uid\\\":\\\"4e3599eacd62834995c77b38ad95f88d\\\",\\\"creatorId\\\":\\\"1199596756811550720\\\",\\\"deviceNmae\\\":\\\"861701069493741\\\",\\\"productId\\\":\\\"yuelan\\\",\\\"productName\\\":\\\"悦蓝cat1穿戴设备\\\"},\\\"messageType\\\":\\\"REPORT_PROPERTY\\\",\\\"properties\\\":{\\\"lat\\\":35.265527067449185,\\\"lng\\\":118.49713144245987,\\\"location\\\":\\\"118.49713144245987,35.265527067449185\\\"},\\\"timestamp\\\":1728719963230}\",createTime=1728719963230i64,id=\"4e3599eacd62834995c77b38ad95f88d\",messageId=\"\",timestamp=1728719963230i64,type=\"reportProperty\" 1728719963230", + "device_log_yuelan_cs1,deviceId=861701069065507 content=\"{\\\"deviceId\\\":\\\"星宇公司-861701069065507\\\",\\\"headers\\\":{\\\"_uid\\\":\\\"9045d6b78b4ffaf1e2d244e912ebbff8\\\",\\\"creatorId\\\":\\\"1199596756811550720\\\",\\\"deviceNmae\\\":\\\"861701069065507\\\",\\\"productId\\\":\\\"yuelan\\\",\\\"productName\\\":\\\"悦蓝cat1穿戴设备\\\"},\\\"messageType\\\":\\\"REPORT_PROPERTY\\\",\\\"properties\\\":{\\\"lat\\\":36.788241914043425,\\\"lng\\\":119.15042325460891,\\\"location\\\":\\\"119.15042325460891,36.788241914043425\\\"},\\\"timestamp\\\":1728719964105}\",createTime=1728719964105i64,id=\"9045d6b78b4ffaf1e2d244e912ebbff8\",messageId=\"\",timestamp=1728719964105i64,type=\"reportProperty\" 1728719964105", + }; + pRes = taos_query(taos, "use ts5528"); + taos_free_result(pRes); + + for( int i = 0; i < 2; i++){ + int32_t totalRows = 0; + pRes = taos_schemaless_insert_raw(taos, sql[i], strlen(sql[i]), &totalRows, TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + int code = taos_errno(pRes); + taos_free_result(pRes); + if (code != 0) { + taos_close(taos); + return code; + } + } + taos_close(taos); + printf("%s result success\n", __FUNCTION__); + return 0; +} + int main(int argc, char *argv[]) { if (argc == 2) { taos_options(TSDB_OPTION_CONFIGDIR, argv[1]); } int ret = 0; + ret = sml_ts5528_test(); + ASSERT(!ret); ret = sml_td29691_Test(); ASSERT(ret); ret = sml_td29373_Test(); From 46995ea6b945aae180812f9324451b7d001a3dc7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 16 Oct 2024 14:36:44 +0800 Subject: [PATCH 39/61] fix: remove invalid error code check and add repair meta function --- source/dnode/mgmt/exe/dmMain.c | 4 + source/dnode/vnode/src/inc/vnodeInt.h | 3 + source/dnode/vnode/src/meta/metaOpen.c | 158 +++++++++++++++++++++++- source/dnode/vnode/src/meta/metaTable.c | 3 - 4 files changed, 163 insertions(+), 5 deletions(-) diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 89569d69d6..0cc2948b5e 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -182,6 +182,8 @@ static void dmSetSignalHandle() { #endif } +extern bool generateNewMeta; + static int32_t dmParseArgs(int32_t argc, char const *argv[]) { global.startTime = taosGetTimestampMs(); @@ -220,6 +222,8 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { global.dumpSdb = true; } else if (strcmp(argv[i], "-dTxn") == 0) { global.deleteTrans = true; + } else if (strcmp(argv[i], "-r") == 0) { + generateNewMeta = true; } else if (strcmp(argv[i], "-E") == 0) { if (i < argc - 1) { if (strlen(argv[++i]) >= PATH_MAX) { diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 1bd4317234..fc98d6578b 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -81,6 +81,9 @@ typedef struct SCommitInfo SCommitInfo; typedef struct SCompactInfo SCompactInfo; typedef struct SQueryNode SQueryNode; +#define VNODE_META_TMP_DIR "meta.tmp" +#define VNODE_META_BACKUP_DIR "meta.backup" + #define VNODE_META_DIR "meta" #define VNODE_TSDB_DIR "tsdb" #define VNODE_TQ_DIR "tq" diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index f062505ac7..615dbce01d 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -133,7 +133,7 @@ static void doScan(SMeta *pMeta) { } } -int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { +static int32_t metaOpenImpl(SVnode *pVnode, SMeta **ppMeta, const char *metaDir, int8_t rollback) { SMeta *pMeta = NULL; int32_t code = 0; int32_t lino; @@ -144,7 +144,11 @@ int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { // create handle vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, path, TSDB_FILENAME_LEN); offset = strlen(path); - snprintf(path + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VNODE_META_DIR); + snprintf(path + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, metaDir); + + if (strncmp(metaDir, VNODE_META_TMP_DIR, strlen(VNODE_META_TMP_DIR)) == 0) { + taosRemoveDir(path); + } if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + strlen(path) + 1)) == NULL) { TSDB_CHECK_CODE(code = terrno, lino, _exit); @@ -245,6 +249,156 @@ _exit: return code; } +bool generateNewMeta = false; + +static int32_t metaGenerateNewMeta(SMeta **ppMeta) { + SMeta *pNewMeta = NULL; + SMeta *pMeta = *ppMeta; + SVnode *pVnode = pMeta->pVnode; + + metaInfo("vgId:%d start to generate new meta", TD_VID(pMeta->pVnode)); + + // Open a new meta for orgainzation + int32_t code = metaOpenImpl(pMeta->pVnode, &pNewMeta, VNODE_META_TMP_DIR, false); + if (code) { + return code; + } + + code = metaBegin(pNewMeta, META_BEGIN_HEAP_NIL); + if (code) { + return code; + } + + // i == 0, scan super table + // i == 1, scan normal table and child table + for (int i = 0; i < 2; i++) { + TBC *uidCursor = NULL; + int32_t counter = 0; + + code = tdbTbcOpen(pMeta->pUidIdx, &uidCursor, NULL); + if (code) { + metaError("vgId:%d failed to open uid index cursor, reason:%s", TD_VID(pVnode), tstrerror(code)); + return code; + } + + code = tdbTbcMoveToFirst(uidCursor); + if (code) { + metaError("vgId:%d failed to move to first, reason:%s", TD_VID(pVnode), tstrerror(code)); + tdbTbcClose(uidCursor); + return code; + } + + for (;;) { + const void *pKey; + int kLen; + const void *pVal; + int vLen; + + if (tdbTbcGet(uidCursor, &pKey, &kLen, &pVal, &vLen) < 0) { + break; + } + + tb_uid_t uid = *(tb_uid_t *)pKey; + SUidIdxVal *pUidIdxVal = (SUidIdxVal *)pVal; + if ((i == 0 && (pUidIdxVal->suid && pUidIdxVal->suid == uid)) // super table + || (i == 1 && (pUidIdxVal->suid == 0 || pUidIdxVal->suid != uid)) // normal table and child table + ) { + counter++; + if (i == 0) { + metaInfo("vgId:%d counter:%d new meta handle %s table uid:%" PRId64, TD_VID(pVnode), counter, "super", uid); + } else { + metaInfo("vgId:%d counter:%d new meta handle %s table uid:%" PRId64, TD_VID(pVnode), counter, + pUidIdxVal->suid == 0 ? "normal" : "child", uid); + } + + // fetch table entry + void *value = NULL; + int valueSize = 0; + if (tdbTbGet(pMeta->pTbDb, + &(STbDbKey){ + .version = pUidIdxVal->version, + .uid = uid, + }, + sizeof(uid), &value, &valueSize) == 0) { + SDecoder dc = {0}; + SMetaEntry me = {0}; + tDecoderInit(&dc, value, valueSize); + if (metaDecodeEntry(&dc, &me) == 0) { + if (metaHandleEntry(pNewMeta, &me) != 0) { + metaError("vgId:%d failed to handle entry, uid:%" PRId64, TD_VID(pVnode), uid); + } + } + tDecoderClear(&dc); + } + tdbFree(value); + } + + code = tdbTbcMoveToNext(uidCursor); + if (code) { + metaError("vgId:%d failed to move to next, reason:%s", TD_VID(pVnode), tstrerror(code)); + return code; + } + } + + tdbTbcClose(uidCursor); + } + + code = metaCommit(pNewMeta, pNewMeta->txn); + if (code) { + metaError("vgId:%d failed to commit, reason:%s", TD_VID(pVnode), tstrerror(code)); + return code; + } + + code = metaFinishCommit(pNewMeta, pNewMeta->txn); + if (code) { + metaError("vgId:%d failed to finish commit, reason:%s", TD_VID(pVnode), tstrerror(code)); + return code; + } + + metaBegin(pNewMeta, META_BEGIN_HEAP_NIL); + metaClose(&pNewMeta); + metaInfo("vgId:%d finish to generate new meta", TD_VID(pVnode)); + return 0; +} + +int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { + int32_t code = metaOpenImpl(pVnode, ppMeta, VNODE_META_DIR, rollback); + if (code) { + return code; + } + + if (generateNewMeta) { + code = metaGenerateNewMeta(ppMeta); + if (code) { + metaError("vgId:%d failed to generate new meta, reason:%s", TD_VID(pVnode), tstrerror(code)); + } + + // backup the old meta + char path[TSDB_FILENAME_LEN] = {0}; + char oldMetaPath[TSDB_FILENAME_LEN] = {0}; + char newMetaPath[TSDB_FILENAME_LEN] = {0}; + char backupMetaPath[TSDB_FILENAME_LEN] = {0}; + + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, path, TSDB_FILENAME_LEN); + snprintf(oldMetaPath, sizeof(oldMetaPath) - 1, "%s%s%s", path, TD_DIRSEP, VNODE_META_DIR); + snprintf(newMetaPath, sizeof(newMetaPath) - 1, "%s%s%s", path, TD_DIRSEP, VNODE_META_TMP_DIR); + snprintf(backupMetaPath, sizeof(backupMetaPath) - 1, "%s%s%s", path, TD_DIRSEP, VNODE_META_BACKUP_DIR); + + metaClose(ppMeta); + taosRenameFile(oldMetaPath, backupMetaPath); + + // rename the new meta to old meta + taosRenameFile(newMetaPath, oldMetaPath); + code = metaOpenImpl(pVnode, ppMeta, VNODE_META_DIR, false); + if (code) { + metaError("vgId:%d failed to open new meta, reason:%s", TD_VID(pVnode), tstrerror(code)); + return code; + } + } + + return TSDB_CODE_SUCCESS; +} + int32_t metaUpgrade(SVnode *pVnode, SMeta **ppMeta) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 08ee422126..21d12ef77d 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -2985,9 +2985,6 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { } } end: - if (terrno != 0) { - ret = terrno; - } tDecoderClear(&dc); tdbFree(pData); return ret; From 39d435722720f6c78ce210deb6cc27cb0ca4adbe Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 16 Oct 2024 15:27:25 +0800 Subject: [PATCH 40/61] make CI pass --- source/dnode/vnode/src/meta/metaOpen.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 615dbce01d..8d22e9732e 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -355,7 +355,9 @@ static int32_t metaGenerateNewMeta(SMeta **ppMeta) { return code; } - metaBegin(pNewMeta, META_BEGIN_HEAP_NIL); + if (metaBegin(pNewMeta, META_BEGIN_HEAP_NIL) != 0) { + metaError("vgId:%d failed to begin new meta", TD_VID(pVnode)); + } metaClose(&pNewMeta); metaInfo("vgId:%d finish to generate new meta", TD_VID(pVnode)); return 0; @@ -385,10 +387,16 @@ int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { snprintf(backupMetaPath, sizeof(backupMetaPath) - 1, "%s%s%s", path, TD_DIRSEP, VNODE_META_BACKUP_DIR); metaClose(ppMeta); - taosRenameFile(oldMetaPath, backupMetaPath); + if (taosRenameFile(oldMetaPath, backupMetaPath) != 0) { + metaError("vgId:%d failed to rename old meta to backup, reason:%s", TD_VID(pVnode), tstrerror(terrno)); + return terrno; + } // rename the new meta to old meta - taosRenameFile(newMetaPath, oldMetaPath); + if (taosRenameFile(newMetaPath, oldMetaPath) != 0) { + metaError("vgId:%d failed to rename new meta to old meta, reason:%s", TD_VID(pVnode), tstrerror(terrno)); + return terrno; + } code = metaOpenImpl(pVnode, ppMeta, VNODE_META_DIR, false); if (code) { metaError("vgId:%d failed to open new meta, reason:%s", TD_VID(pVnode), tstrerror(code)); From e3da1a5b1a517c8450e7fe6dc1b52a0738204c71 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 16 Oct 2024 16:53:02 +0800 Subject: [PATCH 41/61] adjust some code --- source/dnode/vnode/src/meta/metaOpen.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 8d22e9732e..ef36521879 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -355,8 +355,8 @@ static int32_t metaGenerateNewMeta(SMeta **ppMeta) { return code; } - if (metaBegin(pNewMeta, META_BEGIN_HEAP_NIL) != 0) { - metaError("vgId:%d failed to begin new meta", TD_VID(pVnode)); + if ((code = metaBegin(pNewMeta, META_BEGIN_HEAP_NIL)) != 0) { + metaError("vgId:%d failed to begin new meta, reason:%s", TD_VID(pVnode), tstrerror(code)); } metaClose(&pNewMeta); metaInfo("vgId:%d finish to generate new meta", TD_VID(pVnode)); @@ -370,11 +370,6 @@ int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { } if (generateNewMeta) { - code = metaGenerateNewMeta(ppMeta); - if (code) { - metaError("vgId:%d failed to generate new meta, reason:%s", TD_VID(pVnode), tstrerror(code)); - } - // backup the old meta char path[TSDB_FILENAME_LEN] = {0}; char oldMetaPath[TSDB_FILENAME_LEN] = {0}; @@ -386,6 +381,16 @@ int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { snprintf(newMetaPath, sizeof(newMetaPath) - 1, "%s%s%s", path, TD_DIRSEP, VNODE_META_TMP_DIR); snprintf(backupMetaPath, sizeof(backupMetaPath) - 1, "%s%s%s", path, TD_DIRSEP, VNODE_META_BACKUP_DIR); + if (taosCheckExistFile(backupMetaPath)) { + metaError("vgId:%d backup meta already exists, please check", TD_VID(pVnode)); + return TSDB_CODE_FAILED; + } + + code = metaGenerateNewMeta(ppMeta); + if (code) { + metaError("vgId:%d failed to generate new meta, reason:%s", TD_VID(pVnode), tstrerror(code)); + } + metaClose(ppMeta); if (taosRenameFile(oldMetaPath, backupMetaPath) != 0) { metaError("vgId:%d failed to rename old meta to backup, reason:%s", TD_VID(pVnode), tstrerror(terrno)); From 2088969d284fbdd7cc8ff2c6314e8ab407f508f5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 16 Oct 2024 18:56:20 +0800 Subject: [PATCH 42/61] fix: add more check --- source/dnode/vnode/src/meta/metaOpen.c | 77 ++++++++++++++++---------- source/libs/tdb/src/db/tdbBtree.c | 3 + 2 files changed, 51 insertions(+), 29 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index ef36521879..8f2c0b5a5e 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -364,13 +364,7 @@ static int32_t metaGenerateNewMeta(SMeta **ppMeta) { } int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { - int32_t code = metaOpenImpl(pVnode, ppMeta, VNODE_META_DIR, rollback); - if (code) { - return code; - } - if (generateNewMeta) { - // backup the old meta char path[TSDB_FILENAME_LEN] = {0}; char oldMetaPath[TSDB_FILENAME_LEN] = {0}; char newMetaPath[TSDB_FILENAME_LEN] = {0}; @@ -381,32 +375,57 @@ int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { snprintf(newMetaPath, sizeof(newMetaPath) - 1, "%s%s%s", path, TD_DIRSEP, VNODE_META_TMP_DIR); snprintf(backupMetaPath, sizeof(backupMetaPath) - 1, "%s%s%s", path, TD_DIRSEP, VNODE_META_BACKUP_DIR); - if (taosCheckExistFile(backupMetaPath)) { - metaError("vgId:%d backup meta already exists, please check", TD_VID(pVnode)); + bool oldMetaExist = taosCheckExistFile(oldMetaPath); + bool newMetaExist = taosCheckExistFile(newMetaPath); + bool backupMetaExist = taosCheckExistFile(backupMetaPath); + + if ((!backupMetaExist && !oldMetaExist && newMetaExist) // case 2 + || (backupMetaExist && !oldMetaExist && !newMetaExist) // case 4 + || (backupMetaExist && oldMetaExist && newMetaExist) // case 8 + ) { + metaError("vgId:%d invalid meta state, please check", TD_VID(pVnode)); return TSDB_CODE_FAILED; + } else if ((backupMetaExist && oldMetaExist && !newMetaExist) // case 7 + || (!backupMetaExist && !oldMetaExist && !newMetaExist) // case 1 + ) { + return metaOpenImpl(pVnode, ppMeta, VNODE_META_DIR, rollback); + } else if (backupMetaExist && !oldMetaExist && newMetaExist) { + if (taosRenameFile(newMetaPath, oldMetaPath) != 0) { + metaError("vgId:%d failed to rename new meta to old meta, reason:%s", TD_VID(pVnode), tstrerror(terrno)); + return terrno; + } + return metaOpenImpl(pVnode, ppMeta, VNODE_META_DIR, rollback); + } else { + int32_t code = metaOpenImpl(pVnode, ppMeta, VNODE_META_DIR, rollback); + if (code) { + return code; + } + + code = metaGenerateNewMeta(ppMeta); + if (code) { + metaError("vgId:%d failed to generate new meta, reason:%s", TD_VID(pVnode), tstrerror(code)); + } + + metaClose(ppMeta); + if (taosRenameFile(oldMetaPath, backupMetaPath) != 0) { + metaError("vgId:%d failed to rename old meta to backup, reason:%s", TD_VID(pVnode), tstrerror(terrno)); + return terrno; + } + + // rename the new meta to old meta + if (taosRenameFile(newMetaPath, oldMetaPath) != 0) { + metaError("vgId:%d failed to rename new meta to old meta, reason:%s", TD_VID(pVnode), tstrerror(terrno)); + return terrno; + } + code = metaOpenImpl(pVnode, ppMeta, VNODE_META_DIR, false); + if (code) { + metaError("vgId:%d failed to open new meta, reason:%s", TD_VID(pVnode), tstrerror(code)); + return code; + } } - code = metaGenerateNewMeta(ppMeta); - if (code) { - metaError("vgId:%d failed to generate new meta, reason:%s", TD_VID(pVnode), tstrerror(code)); - } - - metaClose(ppMeta); - if (taosRenameFile(oldMetaPath, backupMetaPath) != 0) { - metaError("vgId:%d failed to rename old meta to backup, reason:%s", TD_VID(pVnode), tstrerror(terrno)); - return terrno; - } - - // rename the new meta to old meta - if (taosRenameFile(newMetaPath, oldMetaPath) != 0) { - metaError("vgId:%d failed to rename new meta to old meta, reason:%s", TD_VID(pVnode), tstrerror(terrno)); - return terrno; - } - code = metaOpenImpl(pVnode, ppMeta, VNODE_META_DIR, false); - if (code) { - metaError("vgId:%d failed to open new meta, reason:%s", TD_VID(pVnode), tstrerror(code)); - return code; - } + } else { + return metaOpenImpl(pVnode, ppMeta, VNODE_META_DIR, rollback); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c688a6cc6a..2333a4a6a2 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1446,6 +1446,9 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, return ret; } ofpCell = tdbPageGetCell(ofp, 0); + if (ofpCell == NULL) { + return TSDB_CODE_INVALID_DATA_FMT; + } if (nLeft <= ofp->maxLocal - sizeof(SPgno)) { bytes = nLeft; From 2ae4646cd5bc07c698008f5a6be81e43048f56d0 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 16 Oct 2024 19:41:29 +0800 Subject: [PATCH 43/61] TS-5459: add test case --- tests/system-test/2-query/slow_query_basic.py | 66 +++++++++++++++++++ tests/system-test/win-test-file | 1 + 2 files changed, 67 insertions(+) create mode 100644 tests/system-test/2-query/slow_query_basic.py diff --git a/tests/system-test/2-query/slow_query_basic.py b/tests/system-test/2-query/slow_query_basic.py new file mode 100644 index 0000000000..10595028a7 --- /dev/null +++ b/tests/system-test/2-query/slow_query_basic.py @@ -0,0 +1,66 @@ +import random +import string +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import numpy as np + + +class TDTestCase: + updatecfgDict = {'slowLogThresholdTest': ''} + updatecfgDict["slowLogThresholdTest"] = 0 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if ((tool) in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + paths.append(os.path.join(root, tool)) + break + if (len(paths) == 0): + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] + + def taosBenchmark(self, param): + binPath = self.getPath() + cmd = f"{binPath} {param}" + tdLog.info(cmd) + os.system(cmd) + + def testSlowQuery(self): + self.taosBenchmark(" -d db -t 2 -v 2 -n 1000000 -y") + sql = "select count(*) from db.meters" + for i in range(10): + tdSql.query(sql) + tdSql.checkData(0, 0, 2 * 1000000) + + def run(self): + self.testSlowQuery() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file index 1f2b3f476c..bcb90e3f7d 100644 --- a/tests/system-test/win-test-file +++ b/tests/system-test/win-test-file @@ -926,3 +926,4 @@ python3 ./test.py -f 99-TDcase/TD-20582.py python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3 python3 ./test.py -f 5-taos-tools/taosbenchmark/stt.py -N 3 python3 ./test.py -f eco-system/meta/database/keep_time_offset.py +python3 ./test.py -f 2-query/slow_query_basic.py From 975cb9c907e5ff0e879b3b99dcdc73f2f363b5c3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 10 Oct 2024 15:54:15 +0800 Subject: [PATCH 44/61] fix(stream): add some logs. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 7 ++----- source/libs/stream/src/streamMeta.c | 4 ++++ source/libs/stream/src/streamSched.c | 6 +++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 6b7e857120..3871011407 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -1119,10 +1119,6 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t int32_t vgId = pMeta->vgId; int32_t code = 0; - if (pTask == NULL) { - return -1; - } - streamTaskResume(pTask); ETaskStatus status = streamTaskGetStatus(pTask).state; @@ -1150,7 +1146,6 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t } } - streamMetaReleaseTask(pMeta, pTask); return code; } @@ -1173,6 +1168,7 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode); if (code != 0) { + streamMetaReleaseTask(pMeta, pTask); return code; } @@ -1186,6 +1182,7 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m streamMutexUnlock(&pHTask->lock); code = tqProcessTaskResumeImpl(handle, pHTask, sversion, pReq->igUntreated, fromVnode); + streamMetaReleaseTask(pMeta, pHTask); } return code; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 44c9e76906..29152c6205 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -759,6 +759,10 @@ void streamMetaAcquireOneTask(SStreamTask* pTask) { } void streamMetaReleaseTask(SStreamMeta* UNUSED_PARAM(pMeta), SStreamTask* pTask) { + if (pTask == NULL) { + return; + } + int32_t taskId = pTask->id.taskId; int32_t ref = atomic_sub_fetch_32(&pTask->refCnt, 1); diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index 98920e6f70..095a5af6d4 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -63,7 +63,11 @@ int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int3 pRunReq->reqType = execType; SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; - return tmsgPutToQueue(pMsgCb, STREAM_QUEUE, &msg); + int32_t code = tmsgPutToQueue(pMsgCb, STREAM_QUEUE, &msg); + if (code) { + stError("vgId:%d failed to put msg into stream queue, code:%s, %x", vgId, tstrerror(code), taskId); + } + return code; } void streamTaskClearSchedIdleInfo(SStreamTask* pTask) { pTask->status.schedIdleTime = 0; } From 255a3e9258f139e4c3919e67b3092c0760f9efae Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 10 Oct 2024 16:21:11 +0800 Subject: [PATCH 45/61] fix(util): reset the returned length value. --- source/common/src/tdatablock.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 7a67522231..98e58c8bd7 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2610,7 +2610,10 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf if (code < 0) { uError("func %s failed to convert to ucs charset since %s", __func__, tstrerror(code)); goto _exit; + } else { // reset the length value + code = TSDB_CODE_SUCCESS; } + len += snprintf(dumpBuf + len, size - len, " %15s |", pBuf); if (len >= size - 1) goto _exit; } break; From dfb47ac5a909875efc8688168a917cc0f044e3e8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 10 Oct 2024 18:28:14 +0800 Subject: [PATCH 46/61] fix(stream): only keep the latest pause operation status. --- source/libs/stream/src/streamTaskSm.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 3501d30be4..a10c4c30d5 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -485,6 +485,11 @@ int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, _ static void keepPrevInfo(SStreamTaskSM* pSM) { STaskStateTrans* pTrans = pSM->pActiveTrans; + // we only keep the latest pause state + if (pSM->prev.state.state == TASK_STATUS__PAUSE && pSM->current.state == TASK_STATUS__PAUSE) { + return; + } + pSM->prev.state = pSM->current; pSM->prev.evt = pTrans->event; } @@ -501,9 +506,10 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even if (pTrans == NULL) { ETaskStatus s = pSM->current.state; - if (s != TASK_STATUS__DROPPING && s != TASK_STATUS__PAUSE && s != TASK_STATUS__STOP && - s != TASK_STATUS__UNINIT && s != TASK_STATUS__READY) { - stError("s-task:%s invalid task status:%s on handling event:%s success", id, pSM->current.name, GET_EVT_NAME(pSM->prev.evt)); + if (s != TASK_STATUS__DROPPING && s != TASK_STATUS__PAUSE && s != TASK_STATUS__STOP && s != TASK_STATUS__UNINIT && + s != TASK_STATUS__READY) { + stError("s-task:%s invalid task status:%s on handling event:%s success", id, pSM->current.name, + GET_EVT_NAME(pSM->prev.evt)); } // the pSM->prev.evt may be 0, so print string is not appropriate. From 0c7b3cfb39bd5759c6bf50e2dd9e6b67a1b8456e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 10 Oct 2024 18:34:52 +0800 Subject: [PATCH 47/61] fix(stream): avoid the later pause overwrite the previous pause state. --- source/libs/stream/src/streamTaskSm.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index a10c4c30d5..17d5d884a7 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -485,11 +485,6 @@ int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, _ static void keepPrevInfo(SStreamTaskSM* pSM) { STaskStateTrans* pTrans = pSM->pActiveTrans; - // we only keep the latest pause state - if (pSM->prev.state.state == TASK_STATUS__PAUSE && pSM->current.state == TASK_STATUS__PAUSE) { - return; - } - pSM->prev.state = pSM->current; pSM->prev.evt = pTrans->event; } @@ -527,10 +522,13 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even return TSDB_CODE_STREAM_INVALID_STATETRANS; } - keepPrevInfo(pSM); + // repeat pause will not overwrite the previous pause state + if (pSM->current.state != TASK_STATUS__PAUSE || pTrans->next.state != TASK_STATUS__PAUSE) { + keepPrevInfo(pSM); - pSM->current = pTrans->next; - pSM->pActiveTrans = NULL; + pSM->current = pTrans->next; + pSM->pActiveTrans = NULL; + } // todo remove it // todo: handle the error code From 2a3cb14d2e19f386eaaba267e24b05e6760eceaf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 10 Oct 2024 19:28:08 +0800 Subject: [PATCH 48/61] other: update logs. --- source/libs/stream/src/streamTimer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamTimer.c b/source/libs/stream/src/streamTimer.c index 8b77fe7cb1..0da9acfd1d 100644 --- a/source/libs/stream/src/streamTimer.c +++ b/source/libs/stream/src/streamTimer.c @@ -56,7 +56,7 @@ void streamTmrStart(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* pParam, void* } } - stDebug("vgId:%d start %s tmr succ", vgId, pMsg); + stTrace("vgId:%d start %s tmr succ", vgId, pMsg); } void streamTmrStop(tmr_h tmrId) { From a58371b0ee4fa69b32fef56f8ecd316a73b79d88 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 11 Oct 2024 19:46:54 +0800 Subject: [PATCH 49/61] fix(stream): reset the activeTrans if pause recv repeatly. --- source/libs/stream/src/streamTaskSm.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 17d5d884a7..c3a2742aa2 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -525,11 +525,12 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even // repeat pause will not overwrite the previous pause state if (pSM->current.state != TASK_STATUS__PAUSE || pTrans->next.state != TASK_STATUS__PAUSE) { keepPrevInfo(pSM); - pSM->current = pTrans->next; - pSM->pActiveTrans = NULL; + } else { + stDebug("s-task:%s repeat pause evt recv, not update prev status", id); } + pSM->pActiveTrans = NULL; // todo remove it // todo: handle the error code // on success callback, add into lock if necessary, or maybe we should add an option for this? From 913c7c7831b7a6baca0f1b65ae8aac8cd1f00373 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 16 Oct 2024 18:36:49 +0800 Subject: [PATCH 50/61] refactor: do some internal refactor. --- source/dnode/mnode/impl/src/mndStream.c | 3 ++- source/dnode/mnode/impl/src/mndStreamTransAct.c | 9 +++------ source/dnode/vnode/src/tsdb/tsdbRead2.c | 1 + 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 69d3de25fc..a4327b777f 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1783,7 +1783,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamObj *pStream = NULL; - int32_t code = 0; + int32_t code = 0; if ((code = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { return code; @@ -1811,6 +1811,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { return 0; } + mInfo("stream:%s,%" PRId64 " start to resume stream from pause", resumeReq.name, pStream->uid); if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) { sdbRelease(pMnode->pSdb, pStream); return -1; diff --git a/source/dnode/mnode/impl/src/mndStreamTransAct.c b/source/dnode/mnode/impl/src/mndStreamTransAct.c index 3ecd192222..0f74571b34 100644 --- a/source/dnode/mnode/impl/src/mndStreamTransAct.c +++ b/source/dnode/mnode/impl/src/mndStreamTransAct.c @@ -61,7 +61,6 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) { SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); if (pReq == NULL) { - // terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } @@ -93,7 +92,6 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT if (pReq == NULL) { mError("failed to malloc in resume stream, size:%" PRIzu ", code:%s", sizeof(SVResumeStreamTaskReq), tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - // terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } @@ -106,19 +104,18 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT bool hasEpset = false; int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); if (code != TSDB_CODE_SUCCESS || (!hasEpset)) { - terrno = code; taosMemoryFree(pReq); - return terrno; + return code; } code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0, TSDB_CODE_VND_INVALID_VGROUP_ID); if (code != 0) { taosMemoryFree(pReq); - return terrno; + return code; } mDebug("set the resume action for trans:%d", pTrans->id); - return 0; + return code; } static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask* pTask) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 36bfb56120..990373b87b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -5909,6 +5909,7 @@ int32_t tsdbGetTableSchema(SMeta* pMeta, int64_t uid, STSchema** pSchema, int64_ } else if (mr.me.type == TSDB_NORMAL_TABLE) { // do nothing } else { code = TSDB_CODE_INVALID_PARA; + tsdbError("invalid mr.me.type:%d, code:%s", mr.me.type, tstrerror(code)); metaReaderClear(&mr); return code; } From 4edbb2f91857d93f522b8050cb3b8610320d80fe Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 16 Oct 2024 22:07:37 +0800 Subject: [PATCH 51/61] refactor: do some internal refactor. --- include/libs/stream/tstream.h | 2 +- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +- source/libs/stream/src/streamCheckStatus.c | 2 +- source/libs/stream/src/streamCheckpoint.c | 3 ++- source/libs/stream/src/streamDispatch.c | 2 +- source/libs/stream/src/streamMeta.c | 5 +++-- source/libs/stream/src/streamSched.c | 4 ++-- source/libs/stream/src/streamTask.c | 3 +++ 8 files changed, 14 insertions(+), 9 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index e6d750468e..58c1707e1f 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -754,7 +754,7 @@ int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta); int32_t streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask); int32_t streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); -void streamMetaAcquireOneTask(SStreamTask* pTask); +int32_t streamMetaAcquireOneTask(SStreamTask* pTask); void streamMetaClear(SStreamMeta* pMeta); void streamMetaInitBackend(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta); diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 3871011407..a00e92997c 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -692,7 +692,7 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen STaskId id = {.streamId = pReq->streamId, .taskId = pReq->taskId}; SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if ((ppTask != NULL) && ((*ppTask) != NULL)) { - streamMetaAcquireOneTask(*ppTask); + int32_t unusedRetRef = streamMetaAcquireOneTask(*ppTask); SStreamTask* pTask = *ppTask; if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 75bcc326b3..c1c54b3c0b 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -299,7 +299,7 @@ void streamTaskStartMonitorCheckRsp(SStreamTask* pTask) { return; } - /*SStreamTask* p = */ streamMetaAcquireOneTask(pTask); // add task ref here + int32_t unusedRetRef = streamMetaAcquireOneTask(pTask); // add task ref here streamTaskInitTaskCheckInfo(pInfo, &pTask->outputInfo, taosGetTimestampMs()); int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index e44bca123b..be914d9746 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -347,7 +347,8 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock if (old == 0) { int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s start checkpoint-trigger monitor in 10s, ref:%d ", pTask->id.idStr, ref); - streamMetaAcquireOneTask(pTask); + + int32_t unusedRetRef = streamMetaAcquireOneTask(pTask); streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor"); pTmrInfo->launchChkptId = pActiveInfo->activeId; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 78cbd844a0..5dfcb39cb9 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1158,7 +1158,7 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { if (old == 0) { int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s start checkpoint-ready monitor in 10s, ref:%d ", pTask->id.idStr, ref); - streamMetaAcquireOneTask(pTask); + int32_t unusedRetRef = streamMetaAcquireOneTask(pTask); streamTmrStart(chkptReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor"); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 29152c6205..7e9b60b61a 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -753,9 +753,10 @@ int32_t streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t task return code; } -void streamMetaAcquireOneTask(SStreamTask* pTask) { +int32_t streamMetaAcquireOneTask(SStreamTask* pTask) { int32_t ref = atomic_add_fetch_32(&pTask->refCnt, 1); stTrace("s-task:%s acquire task, ref:%d", pTask->id.idStr, ref); + return ref; } void streamMetaReleaseTask(SStreamMeta* UNUSED_PARAM(pMeta), SStreamTask* pTask) { @@ -866,7 +867,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask) { pTask = *ppTask; - // it is an fill-history task, remove the related stream task's id that points to it + // it is a fill-history task, remove the related stream task's id that points to it if (pTask->info.fillHistory == 0) { int32_t ret = atomic_sub_fetch_32(&pMeta->numOfStreamTasks, 1); } diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index 095a5af6d4..cdaa603e38 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -22,7 +22,7 @@ static void streamTaskSchedHelper(void* param, void* tmrId); void streamSetupScheduleTrigger(SStreamTask* pTask) { int64_t delaySchema = pTask->info.delaySchedParam; if (delaySchema != 0 && pTask->info.fillHistory == 0) { - int32_t ref = atomic_add_fetch_32(&pTask->refCnt, 1); + int32_t ref = streamMetaAcquireOneTask(pTask); stDebug("s-task:%s setup scheduler trigger, ref:%d delay:%" PRId64 " ms", pTask->id.idStr, ref, pTask->info.delaySchedParam); @@ -80,7 +80,7 @@ void streamTaskResumeInFuture(SStreamTask* pTask) { pTask->status.schedIdleTime, ref); // add one ref count for task - streamMetaAcquireOneTask(pTask); + int32_t unusedRetRef = streamMetaAcquireOneTask(pTask); streamTmrStart(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer, &pTask->schedInfo.pIdleTimer, pTask->pMeta->vgId, "resume-task-tmr"); } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 71a2ed3e4a..727701e03e 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -258,10 +258,12 @@ void tFreeStreamTask(SStreamTask* pTask) { if (pTask->inputq.queue) { streamQueueClose(pTask->inputq.queue, pTask->id.taskId); + pTask->inputq.queue = NULL; } if (pTask->outputq.queue) { streamQueueClose(pTask->outputq.queue, pTask->id.taskId); + pTask->outputq.queue = NULL; } if (pTask->exec.qmsg) { @@ -275,6 +277,7 @@ void tFreeStreamTask(SStreamTask* pTask) { if (pTask->exec.pWalReader != NULL) { walCloseReader(pTask->exec.pWalReader); + pTask->exec.pWalReader = NULL; } streamClearChkptReadyMsg(pTask->chkInfo.pActiveInfo); From 15c13cc853ff9f92507d1dba10e389b84d0cc820 Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Wed, 16 Oct 2024 23:35:53 +0800 Subject: [PATCH 52/61] fix:[TD-32592] fix bug when percentile split bucket. --- source/libs/function/src/tpercentile.c | 39 ++++++++++++++------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 29c48460c0..429ab52a8d 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -224,19 +224,18 @@ int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value, int32_t *index *index = -1; - if (v > pBucket->range.dMaxVal || v < pBucket->range.dMinVal) { + if (v > pBucket->range.dMaxVal || v < pBucket->range.dMinVal || isnan(v)) { return TSDB_CODE_SUCCESS; } // divide a range of [dMinVal, dMaxVal] into 1024 buckets double span = pBucket->range.dMaxVal - pBucket->range.dMinVal; - if (span < pBucket->numOfSlots) { - int32_t delta = (int32_t)(v - pBucket->range.dMinVal); - *index = (delta % pBucket->numOfSlots); + if (fabs(span) < DBL_EPSILON) { + *index = 0; } else { double slotSpan = span / pBucket->numOfSlots; *index = (int32_t)((v - pBucket->range.dMinVal) / slotSpan); - if (v == pBucket->range.dMaxVal) { + if (fabs(v - pBucket->range.dMaxVal) < DBL_EPSILON) { *index -= 1; } } @@ -583,48 +582,52 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction *result = getIdenticalDataVal(pMemBucket, i); return TSDB_CODE_SUCCESS; } - // try next round - pMemBucket->times += 1; - // qDebug("MemBucket:%p, start next round data bucketing, time:%d", pMemBucket, pMemBucket->times); - - pMemBucket->range = pSlot->range; - pMemBucket->total = 0; - - resetSlotInfo(pMemBucket); - - int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times - 1); + tMemBucket *tmpBucket = NULL; + int32_t code = tMemBucketCreate(pMemBucket->bytes, pMemBucket->type, pSlot->range.dMinVal, pSlot->range.dMaxVal, + false, &tmpBucket); + if (TSDB_CODE_SUCCESS != code) { + tMemBucketDestroy(&tmpBucket); + return code; + } + int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times); SArray* list; void *p = taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); if (p != NULL) { list = *(SArray **)p; if (list == NULL || list->size <= 0) { + tMemBucketDestroy(&tmpBucket); return -1; } } else { + tMemBucketDestroy(&tmpBucket); return -1; } for (int32_t f = 0; f < list->size; ++f) { int32_t *pageId = taosArrayGet(list, f); if (NULL == pageId) { + tMemBucketDestroy(&tmpBucket); return TSDB_CODE_OUT_OF_RANGE; } SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId); if (pg == NULL) { + tMemBucketDestroy(&tmpBucket); return terrno; } - int32_t code = tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num); + code = tMemBucketPut(tmpBucket, pg->data, (int32_t)pg->num); if (code != TSDB_CODE_SUCCESS) { + tMemBucketDestroy(&tmpBucket); return code; } setBufPageDirty(pg, true); releaseBufPage(pMemBucket->pBuffer, pg); } - - return getPercentileImpl(pMemBucket, count - num, fraction, result); + code = getPercentileImpl(tmpBucket, count - num, fraction, result); + tMemBucketDestroy(&tmpBucket); + return code; } } else { num += pSlot->info.size; From 57cb65273401febe29121c503b3f1723a1631985 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 18 Oct 2024 04:41:22 +0000 Subject: [PATCH 53/61] fix/TS-5532-set-seperate-thread-update-status --- source/dnode/mgmt/mgmt_dnode/inc/dmInt.h | 2 + source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 9 ++++- source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 3 ++ source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 45 +++++++++++++++++++++ 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 18b3f66a60..2d0b9c5a81 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -49,6 +49,7 @@ typedef struct SDnodeMgmt { // dmHandle.c SArray *dmGetMsgHandles(); void dmSendStatusReq(SDnodeMgmt *pMgmt); +void dmUpdateStatusInfo(SDnodeMgmt *pMgmt); void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq); int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); @@ -62,6 +63,7 @@ int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); // dmWorker.c int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmStartStatusThread(SDnodeMgmt *pMgmt); +int32_t dmStartStatusInfoThread(SDnodeMgmt *pMgmt); void dmStopStatusThread(SDnodeMgmt *pMgmt); int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt); void dmStopNotifyThread(SDnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 87b1ae0efa..7be3af0c25 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -22,6 +22,8 @@ extern SConfig *tsCfg; +SMonVloadInfo vinfo = {0}; + static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { int32_t code = 0; if (pMgmt->pData->dnodeId == 0 || pMgmt->pData->clusterId == 0) { @@ -163,8 +165,6 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { (void)taosThreadRwlockUnlock(&pMgmt->pData->lock); dDebug("send status req to mnode, statusSeq:%d, begin to get vnode loads", pMgmt->statusSeq); - SMonVloadInfo vinfo = {0}; - (*pMgmt->getVnodeLoadsFp)(&vinfo); req.pVloads = vinfo.pVloads; dDebug("send status req to mnode, statusSeq:%d, begin to get mnode loads", pMgmt->statusSeq); @@ -231,6 +231,11 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { dmProcessStatusRsp(pMgmt, &rpcRsp); } +void dmUpdateStatusInfo(SDnodeMgmt *pMgmt) { + dDebug("begin to get vnode loads"); + (*pMgmt->getVnodeLoadsFp)(&vinfo); +} + void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq) { int32_t contLen = tSerializeSNotifyReq(NULL, 0, pReq); if (contLen < 0) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 22c2b2f5b2..701aa2d65e 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -22,6 +22,9 @@ static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) { if ((code = dmStartStatusThread(pMgmt)) != 0) { return code; } + if ((code = dmStartStatusInfoThread(pMgmt)) != 0) { + return code; + } #if defined(TD_ENTERPRISE) if ((code = dmStartNotifyThread(pMgmt)) != 0) { return code; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 58b86b20b1..7c9416fac6 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -47,6 +47,35 @@ static void *dmStatusThreadFp(void *param) { return NULL; } +static void *dmStatusInfoThreadFp(void *param) { + SDnodeMgmt *pMgmt = param; + int64_t lastTime = taosGetTimestampMs(); + setThreadName("dnode-status-info"); + + int32_t upTimeCount = 0; + int64_t upTime = 0; + + while (1) { + taosMsleep(200); + if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; + + int64_t curTime = taosGetTimestampMs(); + if (curTime < lastTime) lastTime = curTime; + float interval = (curTime - lastTime) / 1000.0f; + if (interval >= tsStatusInterval) { + dmUpdateStatusInfo(pMgmt); + lastTime = curTime; + + if ((upTimeCount = ((upTimeCount + 1) & 63)) == 0) { + upTime = taosGetOsUptime() - tsDndStartOsUptime; + tsDndUpTime = TMAX(tsDndUpTime, upTime); + } + } + } + + return NULL; +} + SDmNotifyHandle dmNotifyHdl = {.state = 0}; #define TIMESERIES_STASH_NUM 5 static void *dmNotifyThreadFp(void *param) { @@ -280,6 +309,22 @@ int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) { return 0; } +int32_t dmStartStatusInfoThread(SDnodeMgmt *pMgmt) { + int32_t code = 0; + TdThreadAttr thAttr; + (void)taosThreadAttrInit(&thAttr); + (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + if (taosThreadCreate(&pMgmt->statusThread, &thAttr, dmStatusInfoThreadFp, pMgmt) != 0) { + code = TAOS_SYSTEM_ERROR(errno); + dError("failed to create status Info thread since %s", tstrerror(code)); + return code; + } + + (void)taosThreadAttrDestroy(&thAttr); + tmsgReportStartup("dnode-status-info", "initialized"); + return 0; +} + void dmStopStatusThread(SDnodeMgmt *pMgmt) { if (taosCheckPthreadValid(pMgmt->statusThread)) { (void)taosThreadJoin(pMgmt->statusThread, NULL); From 295e4462d5da3ea526a52f4016da0df1f1b7c9df Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 18 Oct 2024 07:23:25 +0000 Subject: [PATCH 54/61] fix/TS-5532-set-seperate-thread-update-status-fix-case --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 13 +++++++++++-- source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 1 + source/dnode/mgmt/node_util/inc/dmUtil.h | 1 + 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 7be3af0c25..1f4a46e887 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -22,7 +22,7 @@ extern SConfig *tsCfg; -SMonVloadInfo vinfo = {0}; +SMonVloadInfo tsVinfo = {0}; static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { int32_t code = 0; @@ -165,7 +165,11 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { (void)taosThreadRwlockUnlock(&pMgmt->pData->lock); dDebug("send status req to mnode, statusSeq:%d, begin to get vnode loads", pMgmt->statusSeq); - req.pVloads = vinfo.pVloads; + (void)taosThreadRwlockRdlock(&pMgmt->pData->statusInfolock); + req.pVloads = taosArrayDup(tsVinfo.pVloads, NULL); + taosArrayDestroy(tsVinfo.pVloads); + tsVinfo.pVloads = NULL; + (void)taosThreadRwlockUnlock(&pMgmt->pData->statusInfolock); dDebug("send status req to mnode, statusSeq:%d, begin to get mnode loads", pMgmt->statusSeq); SMonMloadInfo minfo = {0}; @@ -232,8 +236,13 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { } void dmUpdateStatusInfo(SDnodeMgmt *pMgmt) { + SMonVloadInfo vinfo = {0}; dDebug("begin to get vnode loads"); (*pMgmt->getVnodeLoadsFp)(&vinfo); + (void)taosThreadRwlockWrlock(&pMgmt->pData->statusInfolock); + if (tsVinfo.pVloads == NULL) tsVinfo.pVloads = taosArrayDup(vinfo.pVloads, NULL); + taosArrayDestroy(vinfo.pVloads); + (void)taosThreadRwlockWrlock(&pMgmt->pData->statusInfolock); } void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq) { diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 277dd2e02a..4fc618379c 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -214,6 +214,7 @@ int32_t dmInitVars(SDnode *pDnode) { } (void)taosThreadRwlockInit(&pData->lock, NULL); + (void)taosThreadRwlockInit(&pData->statusInfolock, NULL); (void)taosThreadMutexInit(&pDnode->mutex, NULL); return 0; } diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index b5842acbad..2374c45c2d 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -144,6 +144,7 @@ typedef struct { char machineId[TSDB_MACHINE_ID_LEN + 1]; EEncryptAlgor encryptAlgorigthm; EEncryptScope encryptScope; + TdThreadRwlock statusInfolock; } SDnodeData; typedef struct { From 7e7be90b69004b203e1d246b47872af4900f6980 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 18 Oct 2024 07:51:52 +0000 Subject: [PATCH 55/61] fix/TS-5532-set-seperate-thrad-update-status-fix-case --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 8 ++++---- source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 2 +- source/dnode/mgmt/node_util/inc/dmUtil.h | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 1f4a46e887..08fd632831 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -165,11 +165,11 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { (void)taosThreadRwlockUnlock(&pMgmt->pData->lock); dDebug("send status req to mnode, statusSeq:%d, begin to get vnode loads", pMgmt->statusSeq); - (void)taosThreadRwlockRdlock(&pMgmt->pData->statusInfolock); + (void)taosThreadMutexLock(&pMgmt->pData->statusInfolock); req.pVloads = taosArrayDup(tsVinfo.pVloads, NULL); taosArrayDestroy(tsVinfo.pVloads); tsVinfo.pVloads = NULL; - (void)taosThreadRwlockUnlock(&pMgmt->pData->statusInfolock); + (void)taosThreadMutexUnlock(&pMgmt->pData->statusInfolock); dDebug("send status req to mnode, statusSeq:%d, begin to get mnode loads", pMgmt->statusSeq); SMonMloadInfo minfo = {0}; @@ -239,10 +239,10 @@ void dmUpdateStatusInfo(SDnodeMgmt *pMgmt) { SMonVloadInfo vinfo = {0}; dDebug("begin to get vnode loads"); (*pMgmt->getVnodeLoadsFp)(&vinfo); - (void)taosThreadRwlockWrlock(&pMgmt->pData->statusInfolock); + (void)taosThreadMutexLock(&pMgmt->pData->statusInfolock); if (tsVinfo.pVloads == NULL) tsVinfo.pVloads = taosArrayDup(vinfo.pVloads, NULL); taosArrayDestroy(vinfo.pVloads); - (void)taosThreadRwlockWrlock(&pMgmt->pData->statusInfolock); + (void)taosThreadMutexUnlock(&pMgmt->pData->statusInfolock); } void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq) { diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 4fc618379c..5e4f7163e7 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -214,7 +214,7 @@ int32_t dmInitVars(SDnode *pDnode) { } (void)taosThreadRwlockInit(&pData->lock, NULL); - (void)taosThreadRwlockInit(&pData->statusInfolock, NULL); + (void)taosThreadMutexInit(&pData->statusInfolock, NULL); (void)taosThreadMutexInit(&pDnode->mutex, NULL); return 0; } diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 2374c45c2d..de20f807e9 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -144,7 +144,7 @@ typedef struct { char machineId[TSDB_MACHINE_ID_LEN + 1]; EEncryptAlgor encryptAlgorigthm; EEncryptScope encryptScope; - TdThreadRwlock statusInfolock; + TdThreadMutex statusInfolock; } SDnodeData; typedef struct { From 66e58ee08ff5ef411b5bde632baa4cd4fa7233dc Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 18 Oct 2024 08:36:41 +0000 Subject: [PATCH 56/61] fix/TS-5532-set-seperate-thread-update-status-fix-case --- source/dnode/mgmt/mgmt_dnode/inc/dmInt.h | 2 ++ source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 1 + source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 9 ++++++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 2d0b9c5a81..cbf1959e75 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -28,6 +28,7 @@ typedef struct SDnodeMgmt { const char *path; const char *name; TdThread statusThread; + TdThread statusInfoThread; TdThread notifyThread; TdThread monitorThread; TdThread auditThread; @@ -65,6 +66,7 @@ int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmStartStatusThread(SDnodeMgmt *pMgmt); int32_t dmStartStatusInfoThread(SDnodeMgmt *pMgmt); void dmStopStatusThread(SDnodeMgmt *pMgmt); +void dmStopStatusInfoThread(SDnodeMgmt *pMgmt); int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt); void dmStopNotifyThread(SDnodeMgmt *pMgmt); int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 701aa2d65e..ed156ac1ec 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -47,6 +47,7 @@ static void dmStopMgmt(SDnodeMgmt *pMgmt) { dmStopMonitorThread(pMgmt); dmStopAuditThread(pMgmt); dmStopStatusThread(pMgmt); + dmStopStatusInfoThread(pMgmt); #if defined(TD_ENTERPRISE) dmStopNotifyThread(pMgmt); #endif diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 7c9416fac6..7f802f3837 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -314,7 +314,7 @@ int32_t dmStartStatusInfoThread(SDnodeMgmt *pMgmt) { TdThreadAttr thAttr; (void)taosThreadAttrInit(&thAttr); (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (taosThreadCreate(&pMgmt->statusThread, &thAttr, dmStatusInfoThreadFp, pMgmt) != 0) { + if (taosThreadCreate(&pMgmt->statusInfoThread, &thAttr, dmStatusInfoThreadFp, pMgmt) != 0) { code = TAOS_SYSTEM_ERROR(errno); dError("failed to create status Info thread since %s", tstrerror(code)); return code; @@ -332,6 +332,13 @@ void dmStopStatusThread(SDnodeMgmt *pMgmt) { } } +void dmStopStatusInfoThread(SDnodeMgmt *pMgmt) { + if (taosCheckPthreadValid(pMgmt->statusInfoThread)) { + (void)taosThreadJoin(pMgmt->statusInfoThread, NULL); + taosThreadClear(&pMgmt->statusInfoThread); + } +} + int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt) { int32_t code = 0; TdThreadAttr thAttr; From 707a9fe7eed62e1b19d4d59712acafbb410df55f Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 19 Oct 2024 20:24:19 +0800 Subject: [PATCH 57/61] Update 01-index.md change the link to data model --- docs/zh/01-index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/01-index.md b/docs/zh/01-index.md index df6c327866..32ea117fbb 100644 --- a/docs/zh/01-index.md +++ b/docs/zh/01-index.md @@ -6,7 +6,7 @@ slug: / TDengine 是一款[开源](https://www.taosdata.com/tdengine/open_source_time-series_database)、[高性能](https://www.taosdata.com/fast)、[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database)的时序数据库（Time Series Database, TSDB）, 它专为物联网、车联网、工业互联网、金融、IT 运维等场景优化设计。同时它还带有内建的缓存、流式计算、数据订阅等系统功能，能大幅减少系统设计的复杂度，降低研发和运营成本，是一款极简的时序数据处理平台。本文档是 TDengine 的用户手册，主要是介绍 TDengine 的基本概念、安装、使用、功能、开发接口、运营维护、TDengine 内核设计等等，它主要是面向架构师、开发工程师与系统管理员的。如果你对时序数据的基本概念、价值以及其所能带来的业务价值尚不了解，请参考[时序数据基础](./concept) -TDengine 充分利用了时序数据的特点，提出了“一个数据采集点一张表”与“超级表”的概念，设计了创新的存储引擎，让数据的写入、查询和存储效率都得到极大的提升。为正确理解并使用 TDengine，无论如何，请您仔细阅读[快速入门](./basic)一章。 +TDengine 充分利用了时序数据的特点，提出了“一个数据采集点一张表”与“超级表”的概念，设计了创新的存储引擎，让数据的写入、查询和存储效率都得到极大的提升。为正确理解并使用 TDengine，无论如何，请您仔细阅读[数据模型](./basic/model)一章。 如果你是开发工程师，请一定仔细阅读[开发指南](./develop)一章，该部分对数据库连接、建模、插入数据、查询、流式计算、缓存、数据订阅、用户自定义函数等功能都做了详细介绍，并配有各种编程语言的示例代码。大部分情况下，你只要复制粘贴示例代码，针对自己的应用稍作改动，就能跑起来。对 REST API、各种编程语言的连接器（Connector）想做更多详细了解的话，请看[连接器](./reference/connector)一章。 From 2437030f1a9e68646e065a87cbe191486f823755 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 21 Oct 2024 01:32:18 +0000 Subject: [PATCH 58/61] fix/TS-5532-set-seperate-thread-update-status-fix-review --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 31 +++++++++++++++------ 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 08fd632831..e80cdf0471 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -165,11 +165,16 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { (void)taosThreadRwlockUnlock(&pMgmt->pData->lock); dDebug("send status req to mnode, statusSeq:%d, begin to get vnode loads", pMgmt->statusSeq); - (void)taosThreadMutexLock(&pMgmt->pData->statusInfolock); - req.pVloads = taosArrayDup(tsVinfo.pVloads, NULL); - taosArrayDestroy(tsVinfo.pVloads); + if (taosThreadMutexLock(&pMgmt->pData->statusInfolock) != 0) { + dError("failed to lock status info lock"); + return; + } + taosArraySwap(req.pVloads, tsVinfo.pVloads); tsVinfo.pVloads = NULL; - (void)taosThreadMutexUnlock(&pMgmt->pData->statusInfolock); + if (taosThreadMutexUnlock(&pMgmt->pData->statusInfolock) != 0) { + dError("failed to unlock status info lock"); + return; + } dDebug("send status req to mnode, statusSeq:%d, begin to get mnode loads", pMgmt->statusSeq); SMonMloadInfo minfo = {0}; @@ -239,10 +244,20 @@ void dmUpdateStatusInfo(SDnodeMgmt *pMgmt) { SMonVloadInfo vinfo = {0}; dDebug("begin to get vnode loads"); (*pMgmt->getVnodeLoadsFp)(&vinfo); - (void)taosThreadMutexLock(&pMgmt->pData->statusInfolock); - if (tsVinfo.pVloads == NULL) tsVinfo.pVloads = taosArrayDup(vinfo.pVloads, NULL); - taosArrayDestroy(vinfo.pVloads); - (void)taosThreadMutexUnlock(&pMgmt->pData->statusInfolock); + if (taosThreadMutexLock(&pMgmt->pData->statusInfolock) != 0) { + dError("failed to lock status info lock"); + return; + } + if (tsVinfo.pVloads == NULL) { + taosArraySwap(tsVinfo.pVloads, vinfo.pVloads); + } else { + taosArrayDestroy(vinfo.pVloads); + vinfo.pVloads = NULL; + } + if (taosThreadMutexUnlock(&pMgmt->pData->statusInfolock) != 0) { + dError("failed to unlock status info lock"); + return; + } } void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq) { From 9d2b4eba5a470311367eda712f53ff3aa3354eed Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 21 Oct 2024 03:52:42 +0000 Subject: [PATCH 59/61] fix/TS-5532-set-seperate-thread-update-status-fix-case --- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 7f802f3837..075cd0efce 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -47,6 +47,7 @@ static void *dmStatusThreadFp(void *param) { return NULL; } +extern SMonVloadInfo tsVinfo; static void *dmStatusInfoThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); @@ -73,6 +74,19 @@ static void *dmStatusInfoThreadFp(void *param) { } } + if (taosThreadMutexLock(&pMgmt->pData->statusInfolock) != 0) { + dError("failed to lock status info lock"); + return NULL; + } + if (tsVinfo.pVloads != NULL) { + taosArrayDestroy(tsVinfo.pVloads); + tsVinfo.pVloads = NULL; + } + if (taosThreadMutexUnlock(&pMgmt->pData->statusInfolock) != 0) { + dError("failed to unlock status info lock"); + return NULL; + } + return NULL; } From c454d0f0763af938879aeb2334a96cfdcdafc43d Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 21 Oct 2024 06:19:04 +0000 Subject: [PATCH 60/61] fix/TS-5532-set-seperate-thread-update-status-fix-case --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 6 ++++-- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index e80cdf0471..d0f8fbae87 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -169,7 +169,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { dError("failed to lock status info lock"); return; } - taosArraySwap(req.pVloads, tsVinfo.pVloads); + req.pVloads = tsVinfo.pVloads; tsVinfo.pVloads = NULL; if (taosThreadMutexUnlock(&pMgmt->pData->statusInfolock) != 0) { dError("failed to unlock status info lock"); @@ -244,12 +244,14 @@ void dmUpdateStatusInfo(SDnodeMgmt *pMgmt) { SMonVloadInfo vinfo = {0}; dDebug("begin to get vnode loads"); (*pMgmt->getVnodeLoadsFp)(&vinfo); + dDebug("begin to lock status info"); if (taosThreadMutexLock(&pMgmt->pData->statusInfolock) != 0) { dError("failed to lock status info lock"); return; } if (tsVinfo.pVloads == NULL) { - taosArraySwap(tsVinfo.pVloads, vinfo.pVloads); + tsVinfo.pVloads = vinfo.pVloads; + vinfo.pVloads = NULL; } else { taosArrayDestroy(vinfo.pVloads); vinfo.pVloads = NULL; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 075cd0efce..7fc9920816 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -73,7 +73,7 @@ static void *dmStatusInfoThreadFp(void *param) { } } } - + dDebug("begin to lock status info when thread exit"); if (taosThreadMutexLock(&pMgmt->pData->statusInfolock) != 0) { dError("failed to lock status info lock"); return NULL; From 618cdb9a691cac22cdb73144f593b3506978bfe6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Oct 2024 20:03:58 +0800 Subject: [PATCH 61/61] minor changes --- cmake/taosadapter_CMakeLists.txt.in | 2 +- cmake/taostools_CMakeLists.txt.in | 2 +- source/dnode/mgmt/exe/dmMain.c | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 13826a1a74..ef6ed4af1d 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 9bbda8309f..9a6a5329ae 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index d15fafbda0..040eafbcf1 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -182,7 +182,6 @@ static void dmSetSignalHandle() { } #endif } -extern bool generateNewMeta; extern bool generateNewMeta;