From f686a5b8ae10f9c36fe625bc8d7e98aeb8447340 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Nov 2024 11:22:52 +0800 Subject: [PATCH 01/44] enh: add active info for show grants logs --- source/common/src/systable.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 02efb40f9f..e44ca9cb26 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -437,6 +437,7 @@ static const SSysDbTableSchema userGrantsLogsSchema[] = { {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "active_info", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysDbTableSchema userMachinesSchema[] = { From e3beccbde14f9c5d31911b697ce7115cc5b5b3c3 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 24 Oct 2024 10:50:29 +0800 Subject: [PATCH 02/44] enh: add more log for last cache retrieve --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index d508d75922..c7626dcf36 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -613,6 +613,16 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 singleTableLastTs = pColVal->rowKey.ts; } + if (p->colVal.value.type != pColVal->colVal.value.type) { + // check for type/cid mismatch + tsdbError("last cache type mismatch, uid:%" PRIu64 + ", schema-type:%d, slotId:%d, cache-type:%d, cache-col:%d", + uid, p->colVal.value.type, slotIds[k], pColVal->colVal.value.type, pColVal->colVal.cid); + taosArrayClearEx(pRow, tsdbCacheFreeSLastColItem); + code = TSDB_CODE_INVALID_PARA; + goto _end; + } + if (!IS_VAR_DATA_TYPE(pColVal->colVal.value.type)) { p->colVal = pColVal->colVal; } else { From 5ba9a088d2e149ca4e180953cfa7aba032ce6b8d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 6 Nov 2024 16:15:40 +0800 Subject: [PATCH 03/44] fix(stream): fix the underlying scan operations for stream, when trying to drop stream tasks. --- source/libs/stream/src/streamMeta.c | 8 ++++++++ source/libs/stream/src/streamTaskSm.c | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index f5de719848..9213c729d7 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -887,6 +887,14 @@ static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask, void* param) { tstrerror(code)); } } + + // let's kill the query procedure within stream, to end it ASAP. + if (pTask->info.taskLevel != TASK_LEVEL__SINK && pTask->exec.pExecutor != NULL) { + code = qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:%s failed to kill task related query handle, code:%s", pTask->id.idStr, tstrerror(code)); + } + } return code; } diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index c3a2742aa2..f995c48688 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -500,7 +500,9 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even STaskStateTrans* pTrans = pSM->pActiveTrans; if (pTrans == NULL) { ETaskStatus s = pSM->current.state; - + // when trying to finish current event successfully, another event with high priorities, such as dropping/stop, has + // interrupted this procedure, and changed the status after freeing the activeTrans, resulting in the failure of + // processing of current event. if (s != TASK_STATUS__DROPPING && s != TASK_STATUS__PAUSE && s != TASK_STATUS__STOP && s != TASK_STATUS__UNINIT && s != TASK_STATUS__READY) { stError("s-task:%s invalid task status:%s on handling event:%s success", id, pSM->current.name, From 43a337ef748d3f7187702772f318731107b46e32 Mon Sep 17 00:00:00 2001 From: Jinqing Kuang Date: Tue, 5 Nov 2024 10:57:44 +0800 Subject: [PATCH 04/44] enh(query)[TD-32732]: add client Id generation and usage To address the issue of non-unique identifiers for query tasks, a unique client ID (cid) is now generated during client initialization. This cid is included in every task message sent to the server. The server will use a composite key of to uniquely identify each query task. --- include/common/tmsg.h | 9 + include/libs/executor/executor.h | 2 +- include/libs/nodes/plannodes.h | 1 + include/libs/qworker/qworker.h | 8 +- include/libs/scheduler/scheduler.h | 3 + source/client/src/clientEnv.c | 1 + source/common/src/tmsg.c | 40 ++ source/libs/executor/src/exchangeoperator.c | 67 ++-- source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/nodes/src/nodesCodeFuncs.c | 7 + source/libs/nodes/src/nodesMsgFuncs.c | 6 + source/libs/qworker/inc/qwInt.h | 64 +-- source/libs/qworker/src/qwDbg.c | 8 +- source/libs/qworker/src/qwMsg.c | 13 + source/libs/qworker/src/qwUtil.c | 37 +- source/libs/qworker/src/qworker.c | 14 +- source/libs/scheduler/inc/schInt.h | 38 +- source/libs/scheduler/src/schRemote.c | 14 +- source/libs/scheduler/src/schTask.c | 25 +- source/libs/scheduler/src/schUtil.c | 12 + tests/script/api/sameReqidTest.c | 406 ++++++++++++++++++++ 21 files changed, 650 insertions(+), 126 deletions(-) create mode 100644 tests/script/api/sameReqidTest.c diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7ff70b243a..a7da778513 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2307,6 +2307,7 @@ typedef struct { typedef struct { SExplainRsp rsp; uint64_t qId; + uint64_t cId; uint64_t tId; int64_t rId; int32_t eId; @@ -2660,6 +2661,7 @@ typedef struct SSubQueryMsg { SMsgHead header; uint64_t sId; uint64_t queryId; + uint64_t clientId; uint64_t taskId; int64_t refId; int32_t execId; @@ -2689,6 +2691,7 @@ typedef struct { SMsgHead header; uint64_t sId; uint64_t queryId; + uint64_t clientId; uint64_t taskId; int32_t execId; } SQueryContinueReq; @@ -2723,6 +2726,7 @@ typedef struct { SMsgHead header; uint64_t sId; uint64_t queryId; + uint64_t clientId; uint64_t taskId; int32_t execId; SOperatorParam* pOpParam; @@ -2738,6 +2742,7 @@ typedef struct { typedef struct { uint64_t queryId; + uint64_t clientId; uint64_t taskId; int64_t refId; int32_t execId; @@ -2784,6 +2789,7 @@ typedef struct { SMsgHead header; uint64_t sId; uint64_t queryId; + uint64_t clientId; uint64_t taskId; int64_t refId; int32_t execId; @@ -2797,6 +2803,7 @@ typedef struct { SMsgHead header; uint64_t sId; uint64_t queryId; + uint64_t clientId; uint64_t taskId; int64_t refId; int32_t execId; @@ -2813,6 +2820,7 @@ typedef struct { SMsgHead header; uint64_t sId; uint64_t queryId; + uint64_t clientId; uint64_t taskId; int64_t refId; int32_t execId; @@ -4261,6 +4269,7 @@ typedef struct { SMsgHead header; uint64_t sId; uint64_t queryId; + uint64_t clientId; uint64_t taskId; uint32_t sqlLen; uint32_t phyLen; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index d955a7b3b9..82cb899cb6 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -31,7 +31,7 @@ typedef void* DataSinkHandle; struct SRpcMsg; struct SSubplan; -typedef int32_t (*localFetchFp)(void*, uint64_t, uint64_t, uint64_t, int64_t, int32_t, void**, SArray*); +typedef int32_t (*localFetchFp)(void*, uint64_t, uint64_t, uint64_t, uint64_t, int64_t, int32_t, void**, SArray*); typedef struct { void* handle; diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index cfd9c1a422..48852e5552 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -624,6 +624,7 @@ typedef struct SAggPhysiNode { typedef struct SDownstreamSourceNode { ENodeType type; SQueryNodeAddr addr; + uint64_t clientId; uint64_t taskId; uint64_t schedId; int32_t execId; diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index 83daf0376c..cb4e359727 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -105,11 +105,11 @@ void qWorkerDestroy(void **qWorkerMgmt); int32_t qWorkerGetStat(SReadHandle *handle, void *qWorkerMgmt, SQWorkerStat *pStat); -int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId, - SQWMsg *qwMsg, SArray *explainRes); +int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t cId, uint64_t tId, int64_t rId, + int32_t eId, SQWMsg *qwMsg, SArray *explainRes); -int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId, - void **pRsp, SArray *explainRes); +int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t cId, uint64_t tId, int64_t rId, + int32_t eId, void **pRsp, SArray *explainRes); int32_t qWorkerDbgEnableDebug(char *option); diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index af8deff1a0..2988ffc4b1 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -83,6 +83,9 @@ void schedulerStopQueryHb(void* pTrans); int32_t schedulerUpdatePolicy(int32_t policy); int32_t schedulerEnableReSchedule(bool enableResche); +int32_t initClientId(void); +uint64_t getClientId(void); + /** * Cancel query job * @param pJob diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index c56a627ec7..fa9df5be73 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -983,6 +983,7 @@ void taos_init_imp(void) { SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100}; ENV_ERR_RET(catalogInit(&cfg), "failed to init catalog"); ENV_ERR_RET(schedulerInit(), "failed to init scheduler"); + ENV_ERR_RET(initClientId(), "failed to init clientId"); tscDebug("starting to initialize TAOS driver"); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 458badc764..8b837a2dc3 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -8717,6 +8717,7 @@ int32_t tSerializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) { TAOS_CHECK_EXIT(tEncodeCStrWithLen(&encoder, pReq->sql, pReq->sqlLen)); TAOS_CHECK_EXIT(tEncodeU32(&encoder, pReq->msgLen)); TAOS_CHECK_EXIT(tEncodeBinary(&encoder, (uint8_t *)pReq->msg, pReq->msgLen)); + TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->clientId)); tEndEncode(&encoder); @@ -8765,6 +8766,11 @@ int32_t tDeserializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) TAOS_CHECK_EXIT(tDecodeCStrAlloc(&decoder, &pReq->sql)); TAOS_CHECK_EXIT(tDecodeU32(&decoder, &pReq->msgLen)); TAOS_CHECK_EXIT(tDecodeBinaryAlloc(&decoder, (void **)&pReq->msg, NULL)); + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->clientId)); + } else { + pReq->clientId = 0; + } tEndDecode(&decoder); @@ -8894,6 +8900,7 @@ int32_t tSerializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq) { } else { TAOS_CHECK_EXIT(tEncodeI32(&encoder, 0)); } + TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->clientId)); tEndEncode(&encoder); @@ -8943,6 +8950,11 @@ int32_t tDeserializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq) } TAOS_CHECK_EXIT(tDeserializeSOperatorParam(&decoder, pReq->pOpParam)); } + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->clientId)); + } else { + pReq->clientId = 0; + } tEndDecode(&decoder); @@ -9055,6 +9067,7 @@ int32_t tSerializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) { TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->taskId)); TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->refId)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->execId)); + TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->clientId)); tEndEncode(&encoder); @@ -9095,6 +9108,11 @@ int32_t tDeserializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->taskId)); TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->refId)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->execId)); + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->clientId)); + } else { + pReq->clientId = 0; + } tEndDecode(&decoder); @@ -9123,6 +9141,7 @@ int32_t tSerializeSTaskNotifyReq(void *buf, int32_t bufLen, STaskNotifyReq *pReq TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->refId)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->execId)); TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->type)); + TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->clientId)); tEndEncode(&encoder); @@ -9164,6 +9183,11 @@ int32_t tDeserializeSTaskNotifyReq(void *buf, int32_t bufLen, STaskNotifyReq *pR TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->refId)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->execId)); TAOS_CHECK_EXIT(tDecodeI32(&decoder, (int32_t *)&pReq->type)); + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->clientId)); + } else { + pReq->clientId = 0; + } tEndDecode(&decoder); @@ -9353,6 +9377,10 @@ int32_t tSerializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *pR TAOS_CHECK_EXIT(tEncodeI32(&encoder, status->execId)); TAOS_CHECK_EXIT(tEncodeI8(&encoder, status->status)); } + for (int32_t i = 0; i < num; ++i) { + STaskStatus *status = taosArrayGet(pRsp->taskStatus, i); + TAOS_CHECK_EXIT(tEncodeU64(&encoder, status->clientId)); + } } else { TAOS_CHECK_EXIT(tEncodeI32(&encoder, 0)); } @@ -9396,6 +9424,12 @@ int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp * TAOS_CHECK_EXIT(terrno); } } + if (!tDecodeIsEnd(&decoder)) { + for (int32_t i = 0; i < num; ++i) { + STaskStatus *status = taosArrayGet(pRsp->taskStatus, i); + TAOS_CHECK_EXIT(tDecodeU64(&decoder, &status->clientId)); + } + } } else { pRsp->taskStatus = NULL; } @@ -9560,6 +9594,7 @@ int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->sql)); TAOS_CHECK_EXIT(tEncodeBinary(&encoder, pReq->msg, pReq->phyLen)); TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->source)); + TAOS_CHECK_EXIT(tEncodeU64(&encoder, pReq->clientId)); tEndEncode(&encoder); _exit: @@ -9608,6 +9643,11 @@ int32_t tDeserializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { if (!tDecodeIsEnd(&decoder)) { TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->source)); } + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeU64(&decoder, &pReq->clientId)); + } else { + pReq->clientId = 0; + } tEndDecode(&decoder); _exit: diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 042fcf0120..7222f2d297 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -121,10 +121,10 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn } } else { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows, - pExchangeInfo->loadInfo.totalRows, i + 1, totalSources); + qDebug("%s vgId:%d, clientId:0x%" PRIx64 " taskId:0x%" PRIx64 + " execId:%d index:%d completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId, i, + pDataInfo->totalRows, pExchangeInfo->loadInfo.totalRows, i + 1, totalSources); taosMemoryFreeClear(pDataInfo->pRsp); } break; @@ -141,17 +141,17 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn if (pRsp->completed == 1) { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 + qDebug("%s fetch msg rsp from vgId:%d, clientId:0x%" PRIx64 " taskId:0x%" PRIx64 " execId:%d index:%d completed, blocks:%d, numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", total:%.2f Kb, try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, - pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, i + 1, - totalSources); + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId, i, + pRsp->numOfBlocks, pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, + pLoadInfo->totalSize / 1024.0, i + 1, totalSources); } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d blocks:%d, numOfRows:%" PRId64 - ", totalRows:%" PRIu64 ", total:%.2f Kb", - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks, - pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); + qDebug("%s fetch msg rsp from vgId:%d, clientId:0x%" PRIx64 " taskId:0x%" PRIx64 + " execId:%d blocks:%d, numOfRows:%" PRId64 ", totalRows:%" PRIu64 ", total:%.2f Kb", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId, + pRsp->numOfBlocks, pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); } taosMemoryFreeClear(pDataInfo->pRsp); @@ -640,9 +640,9 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas if (pSource->localExec) { SDataBuf pBuf = {0}; - int32_t code = - (*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId, - pSource->taskId, 0, pSource->execId, &pBuf.pData, pTaskInfo->localFetch.explainRes); + int32_t code = (*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId, + pSource->clientId, pSource->taskId, 0, pSource->execId, &pBuf.pData, + pTaskInfo->localFetch.explainRes); code = loadRemoteDataCallback(pWrapper, &pBuf, code); QUERY_CHECK_CODE(code, lino, _end); taosMemoryFree(pWrapper); @@ -650,6 +650,7 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas SResFetchReq req = {0}; req.header.vgId = pSource->addr.nodeId; req.sId = pSource->schedId; + req.clientId = pSource->clientId; req.taskId = pSource->taskId; req.queryId = pTaskInfo->id.queryId; req.execId = pSource->execId; @@ -691,9 +692,10 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas freeOperatorParam(req.pOpParam, OP_GET_PARAM); - qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %p, %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, - pSource->execId, pExchangeInfo, sourceIndex, totalSources); + qDebug("%s build fetch msg and send to vgId:%d, ep:%s, clientId:0x%" PRIx64 " taskId:0x%" PRIx64 + ", execId:%d, %p, %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->clientId, + pSource->taskId, pSource->execId, pExchangeInfo, sourceIndex, totalSources); // send the fetch remote task result reques SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); @@ -974,8 +976,9 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { } if (pDataInfo->code != TSDB_CODE_SUCCESS) { - qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), - pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code)); + qError("%s vgId:%d, clientId:0x%" PRIx64 " taskID:0x%" PRIx64 " execId:%d error happens, code:%s", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId, + tstrerror(pDataInfo->code)); pOperator->pTaskInfo->code = pDataInfo->code; return pOperator->pTaskInfo->code; } @@ -984,10 +987,10 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; if (pRsp->numOfRows == 0) { - qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 " try next", - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1, - pDataInfo->totalRows, pLoadInfo->totalRows); + qDebug("%s vgId:%d, clientId:0x%" PRIx64 " taskID:0x%" PRIx64 + " execId:%d %d of total completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 " try next", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId, + pExchangeInfo->current + 1, pDataInfo->totalRows, pLoadInfo->totalRows); pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; pExchangeInfo->current += 1; @@ -1002,19 +1005,19 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; if (pRsp->completed == 1) { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 + qDebug("%s fetch msg rsp from vgId:%d, clientId:0x%" PRIx64 " taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, - pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, - totalSources); + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId, + pRetrieveRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, + pExchangeInfo->current + 1, totalSources); pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; pExchangeInfo->current += 1; } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 ", totalRows:%" PRIu64 - ", totalBytes:%" PRIu64, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, - pLoadInfo->totalRows, pLoadInfo->totalSize); + qDebug("%s fetch msg rsp from vgId:%d, clientId:0x%" PRIx64 " taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 + ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->clientId, pSource->taskId, pSource->execId, + pRetrieveRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize); } updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 1a5785190b..ba87912670 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -851,6 +851,7 @@ static int32_t slotDescCopy(const SSlotDescNode* pSrc, SSlotDescNode* pDst) { static int32_t downstreamSourceCopy(const SDownstreamSourceNode* pSrc, SDownstreamSourceNode* pDst) { COPY_OBJECT_FIELD(addr, sizeof(SQueryNodeAddr)); + COPY_SCALAR_FIELD(clientId); COPY_SCALAR_FIELD(taskId); COPY_SCALAR_FIELD(schedId); COPY_SCALAR_FIELD(execId); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 3275cfd838..f7f858db78 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -5259,6 +5259,7 @@ static int32_t jsonToColumnDefNode(const SJson* pJson, void* pObj) { } static const char* jkDownstreamSourceAddr = "Addr"; +static const char* jkDownstreamSourceClientId = "ClientId"; static const char* jkDownstreamSourceTaskId = "TaskId"; static const char* jkDownstreamSourceSchedId = "SchedId"; static const char* jkDownstreamSourceExecId = "ExecId"; @@ -5268,6 +5269,9 @@ static int32_t downstreamSourceNodeToJson(const void* pObj, SJson* pJson) { const SDownstreamSourceNode* pNode = (const SDownstreamSourceNode*)pObj; int32_t code = tjsonAddObject(pJson, jkDownstreamSourceAddr, queryNodeAddrToJson, &pNode->addr); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceClientId, pNode->clientId); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkDownstreamSourceTaskId, pNode->taskId); } @@ -5288,6 +5292,9 @@ static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) { SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)pObj; int32_t code = tjsonToObject(pJson, jkDownstreamSourceAddr, jsonToQueryNodeAddr, &pNode->addr); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUBigIntValue(pJson, jkDownstreamSourceClientId, &pNode->clientId); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetUBigIntValue(pJson, jkDownstreamSourceTaskId, &pNode->taskId); } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 28d0b9fbd4..bf3ea66e47 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -1769,6 +1769,9 @@ static int32_t downstreamSourceNodeInlineToMsg(const void* pObj, STlvEncoder* pE if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeValueI32(pEncoder, pNode->fetchMsgType); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueU64(pEncoder, pNode->clientId); + } return code; } @@ -1793,6 +1796,9 @@ static int32_t msgToDownstreamSourceNodeInlineToMsg(STlvDecoder* pDecoder, void* if (TSDB_CODE_SUCCESS == code) { code = tlvDecodeValueI32(pDecoder, &pNode->fetchMsgType); } + if (TSDB_CODE_SUCCESS == code && !tlvDecodeEnd(pDecoder)) { + code = tlvDecodeValueU64(pDecoder, &pNode->clientId); + } return code; } diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 7a902bdd66..708c285aea 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -215,8 +215,8 @@ typedef struct SQWorkerMgmt { #define QW_CTX_NOT_EXISTS_ERR_CODE(mgmt) \ (atomic_load_8(&(mgmt)->nodeStopped) ? TSDB_CODE_VND_STOPPED : TSDB_CODE_QRY_TASK_CTX_NOT_EXIST) -#define QW_FPARAMS_DEF SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId -#define QW_IDS() sId, qId, tId, rId, eId +#define QW_FPARAMS_DEF SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t cId, uint64_t tId, int64_t rId, int32_t eId +#define QW_IDS() sId, qId, cId, tId, rId, eId #define QW_FPARAMS() mgmt, QW_IDS() #define QW_STAT_INC(_item, _n) (void)atomic_add_fetch_64(&(_item), _n) @@ -257,18 +257,20 @@ typedef struct SQWorkerMgmt { #define QW_FETCH_RUNNING(ctx) ((ctx)->inFetch) #define QW_QUERY_NOT_STARTED(ctx) (QW_GET_PHASE(ctx) == -1) -#define QW_SET_QTID(id, qId, tId, eId) \ - do { \ - *(uint64_t *)(id) = (qId); \ - *(uint64_t *)((char *)(id) + sizeof(qId)) = (tId); \ - *(int32_t *)((char *)(id) + sizeof(qId) + sizeof(tId)) = (eId); \ +#define QW_SET_QTID(id, qId, cId, tId, eId) \ + do { \ + *(uint64_t *)(id) = (qId); \ + *(uint64_t *)((char *)(id) + sizeof(qId)) = (cId); \ + *(uint64_t *)((char *)(id) + sizeof(qId) + sizeof(cId)) = (tId); \ + *(int32_t *)((char *)(id) + sizeof(qId) + sizeof(cId) + sizeof(tId)) = (eId); \ } while (0) -#define QW_GET_QTID(id, qId, tId, eId) \ - do { \ - (qId) = *(uint64_t *)(id); \ - (tId) = *(uint64_t *)((char *)(id) + sizeof(qId)); \ - (eId) = *(int32_t *)((char *)(id) + sizeof(qId) + sizeof(tId)); \ +#define QW_GET_QTID(id, qId, cId, tId, eId) \ + do { \ + (qId) = *(uint64_t *)(id); \ + (cId) = *(uint64_t *)((char *)(id) + sizeof(qId)); \ + (tId) = *(uint64_t *)((char *)(id) + sizeof(qId) + sizeof(cId)); \ + (eId) = *(int32_t *)((char *)(id) + sizeof(qId) + sizeof(cId) + sizeof(tId)); \ } while (0) #define QW_ERR_RET(c) \ @@ -310,25 +312,31 @@ typedef struct SQWorkerMgmt { #define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%" PRIx64 " " param, mgmt, sId, __VA_ARGS__) #define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%" PRIx64 " " param, mgmt, sId, __VA_ARGS__) -#define QW_TASK_ELOG(param, ...) qError("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__) -#define QW_TASK_WLOG(param, ...) qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__) -#define QW_TASK_DLOG(param, ...) qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__) +#define QW_TASK_ELOG(param, ...) \ + qError("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId, __VA_ARGS__) +#define QW_TASK_WLOG(param, ...) \ + qWarn("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId, __VA_ARGS__) +#define QW_TASK_DLOG(param, ...) \ + qDebug("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId, __VA_ARGS__) #define QW_TASK_DLOGL(param, ...) \ - qDebugL("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId, __VA_ARGS__) + qDebugL("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId, __VA_ARGS__) -#define QW_TASK_ELOG_E(param) qError("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId) -#define QW_TASK_WLOG_E(param) qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId) -#define QW_TASK_DLOG_E(param) qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, tId, eId) +#define QW_TASK_ELOG_E(param) \ + qError("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId) +#define QW_TASK_WLOG_E(param) \ + qWarn("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId) +#define QW_TASK_DLOG_E(param) \ + qDebug("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, qId, cId, tId, eId) -#define QW_SCH_TASK_ELOG(param, ...) \ - qError("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, tId, eId, \ - __VA_ARGS__) -#define QW_SCH_TASK_WLOG(param, ...) \ - qWarn("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, tId, eId, \ - __VA_ARGS__) -#define QW_SCH_TASK_DLOG(param, ...) \ - qDebug("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, tId, eId, \ - __VA_ARGS__) +#define QW_SCH_TASK_ELOG(param, ...) \ + qError("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, \ + qId, cId, tId, eId, __VA_ARGS__) +#define QW_SCH_TASK_WLOG(param, ...) \ + qWarn("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, qId, \ + cId, tId, eId, __VA_ARGS__) +#define QW_SCH_TASK_DLOG(param, ...) \ + qDebug("QW:%p SID:0x%" PRIx64 ",qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, mgmt, sId, \ + qId, cId, tId, eId, __VA_ARGS__) #define QW_LOCK_DEBUG(...) \ do { \ diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index d3b8d36b25..897080df3e 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -96,14 +96,14 @@ void qwDbgDumpSchInfo(SQWorker *mgmt, SQWSchStatus *sch, int32_t i) { int32_t taskNum = taosHashGetSize(sch->tasksHash); QW_DLOG("***The %dth scheduler status, hbBrokenTs:%" PRId64 ",taskNum:%d", i, sch->hbBrokenTs, taskNum); - uint64_t qId, tId; + uint64_t qId, cId, tId; int32_t eId; SQWTaskStatus *pTask = NULL; void *pIter = taosHashIterate(sch->tasksHash, NULL); while (pIter) { pTask = (SQWTaskStatus *)pIter; void *key = taosHashGetKey(pIter, NULL); - QW_GET_QTID(key, qId, tId, eId); + QW_GET_QTID(key, qId, cId, tId, eId); QW_TASK_DLOG("job refId:%" PRIx64 ", code:%x, task status:%d", pTask->refId, pTask->code, pTask->status); @@ -118,13 +118,13 @@ void qwDbgDumpTasksInfo(SQWorker *mgmt) { int32_t i = 0; SQWTaskCtx *ctx = NULL; - uint64_t qId, tId; + uint64_t qId, cId, tId; int32_t eId; void *pIter = taosHashIterate(mgmt->ctxHash, NULL); while (pIter) { ctx = (SQWTaskCtx *)pIter; void *key = taosHashGetKey(pIter, NULL); - QW_GET_QTID(key, qId, tId, eId); + QW_GET_QTID(key, qId, cId, tId, eId); QW_TASK_DLOG("%p lock:%x, phase:%d, type:%d, explain:%d, needFetch:%d, localExec:%d, queryMsgType:%d, " "sId:%" PRId64 ", level:%d, queryGotData:%d, queryRsped:%d, queryEnd:%d, queryContinue:%d, queryInQueue:%d, " diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 20b81bfc14..7dbad90cc0 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -233,6 +233,7 @@ int32_t qwBuildAndSendDropMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { qMsg.header.contLen = 0; qMsg.sId = sId; qMsg.queryId = qId; + qMsg.clientId = cId; qMsg.taskId = tId; qMsg.refId = rId; qMsg.execId = eId; @@ -284,6 +285,7 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { req->header.vgId = mgmt->nodeId; req->sId = sId; req->queryId = qId; + req->clientId = cId; req->taskId = tId; req->execId = eId; @@ -312,6 +314,7 @@ int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { qMsg.header.contLen = 0; qMsg.sId = sId; qMsg.queryId = qId; + qMsg.clientId = cId; qMsg.taskId = tId; qMsg.refId = rId; qMsg.execId = eId; @@ -416,6 +419,7 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg, bool chkGran uint64_t sId = msg.sId; uint64_t qId = msg.queryId; + uint64_t cId = msg.clientId; uint64_t tId = msg.taskId; int64_t rId = msg.refId; int32_t eId = msg.execId; @@ -447,6 +451,7 @@ int32_t qWorkerAbortPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg) { uint64_t sId = msg.sId; uint64_t qId = msg.queryId; + uint64_t cId = msg.clientId; uint64_t tId = msg.taskId; int64_t rId = msg.refId; int32_t eId = msg.execId; @@ -479,6 +484,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int uint64_t sId = msg.sId; uint64_t qId = msg.queryId; + uint64_t cId = msg.clientId; uint64_t tId = msg.taskId; int64_t rId = msg.refId; int32_t eId = msg.execId; @@ -524,6 +530,7 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in uint64_t sId = msg->sId; uint64_t qId = msg->queryId; + uint64_t cId = msg->clientId; uint64_t tId = msg->taskId; int64_t rId = 0; int32_t eId = msg->execId; @@ -557,6 +564,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int uint64_t sId = req.sId; uint64_t qId = req.queryId; + uint64_t cId = req.clientId; uint64_t tId = req.taskId; int64_t rId = 0; int32_t eId = req.execId; @@ -604,12 +612,14 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in msg->sId = be64toh(msg->sId); msg->queryId = be64toh(msg->queryId); + msg->clientId = be64toh(msg->clientId); msg->taskId = be64toh(msg->taskId); msg->refId = be64toh(msg->refId); msg->execId = ntohl(msg->execId); uint64_t sId = msg->sId; uint64_t qId = msg->queryId; + uint64_t cId = msg->clientId; uint64_t tId = msg->taskId; int64_t rId = msg->refId; int32_t eId = msg->execId; @@ -646,6 +656,7 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int6 uint64_t sId = msg.sId; uint64_t qId = msg.queryId; + uint64_t cId = msg.clientId; uint64_t tId = msg.taskId; int64_t rId = msg.refId; int32_t eId = msg.execId; @@ -684,6 +695,7 @@ int32_t qWorkerProcessNotifyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in uint64_t sId = msg.sId; uint64_t qId = msg.queryId; + uint64_t cId = msg.clientId; uint64_t tId = msg.taskId; int64_t rId = msg.refId; int32_t eId = msg.execId; @@ -753,6 +765,7 @@ int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SD uint64_t sId = req.sId; uint64_t qId = req.queryId; + uint64_t cId = req.clientId; uint64_t tId = req.taskId; int64_t rId = 0; int32_t eId = -1; diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index ef07a42629..917579deb0 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -137,8 +137,8 @@ int32_t qwAcquireScheduler(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchS void qwReleaseScheduler(int32_t rwType, SQWorker *mgmt) { QW_UNLOCK(rwType, &mgmt->schLock); } int32_t qwAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch, SQWTaskStatus **task) { - char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0}; - QW_SET_QTID(id, qId, tId, eId); + char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; + QW_SET_QTID(id, qId, cId, tId, eId); QW_LOCK(rwType, &sch->tasksLock); *task = taosHashGet(sch->tasksHash, id, sizeof(id)); @@ -153,8 +153,8 @@ int32_t qwAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch, S int32_t qwAddTaskStatusImpl(QW_FPARAMS_DEF, SQWSchStatus *sch, int32_t rwType, int32_t status, SQWTaskStatus **task) { int32_t code = 0; - char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0}; - QW_SET_QTID(id, qId, tId, eId); + char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; + QW_SET_QTID(id, qId, cId, tId, eId); SQWTaskStatus ntask = {0}; ntask.status = status; @@ -209,8 +209,8 @@ int32_t qwAddAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch void qwReleaseTaskStatus(int32_t rwType, SQWSchStatus *sch) { QW_UNLOCK(rwType, &sch->tasksLock); } int32_t qwAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { - char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0}; - QW_SET_QTID(id, qId, tId, eId); + char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; + QW_SET_QTID(id, qId, cId, tId, eId); *ctx = taosHashAcquire(mgmt->ctxHash, id, sizeof(id)); if (NULL == (*ctx)) { @@ -222,8 +222,8 @@ int32_t qwAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { } int32_t qwGetTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { - char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0}; - QW_SET_QTID(id, qId, tId, eId); + char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; + QW_SET_QTID(id, qId, cId, tId, eId); *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); if (NULL == (*ctx)) { @@ -235,8 +235,8 @@ int32_t qwGetTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { } int32_t qwAddTaskCtxImpl(QW_FPARAMS_DEF, bool acquire, SQWTaskCtx **ctx) { - char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0}; - QW_SET_QTID(id, qId, tId, eId); + char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; + QW_SET_QTID(id, qId, cId, tId, eId); SQWTaskCtx nctx = {0}; @@ -347,6 +347,7 @@ int32_t qwSendExplainResponse(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { (void)memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo)); localRsp.rsp.subplanInfo = pExec; localRsp.qId = qId; + localRsp.cId = cId; localRsp.tId = tId; localRsp.rId = rId; localRsp.eId = eId; @@ -376,8 +377,8 @@ _return: int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { - char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0}; - QW_SET_QTID(id, qId, tId, eId); + char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; + QW_SET_QTID(id, qId, cId, tId, eId); SQWTaskCtx octx; SQWTaskCtx *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); @@ -411,8 +412,8 @@ int32_t qwDropTaskStatus(QW_FPARAMS_DEF) { SQWTaskStatus *task = NULL; int32_t code = 0; - char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0}; - QW_SET_QTID(id, qId, tId, eId); + char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; + QW_SET_QTID(id, qId, cId, tId, eId); if (qwAcquireScheduler(mgmt, sId, QW_WRITE, &sch)) { QW_TASK_WLOG_E("scheduler does not exist"); @@ -465,8 +466,8 @@ _return: int32_t qwHandleDynamicTaskEnd(QW_FPARAMS_DEF) { - char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0}; - QW_SET_QTID(id, qId, tId, eId); + char id[sizeof(qId) + sizeof(cId) + sizeof(tId) + sizeof(eId)] = {0}; + QW_SET_QTID(id, qId, cId, tId, eId); SQWTaskCtx octx; SQWTaskCtx *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); @@ -588,14 +589,14 @@ void qwDestroyImpl(void *pMgmt) { mgmt->hbTimer = NULL; taosTmrCleanUp(mgmt->timer); - uint64_t qId, tId; + uint64_t qId, cId, tId; int32_t eId; void *pIter = taosHashIterate(mgmt->ctxHash, NULL); while (pIter) { SQWTaskCtx *ctx = (SQWTaskCtx *)pIter; void *key = taosHashGetKey(pIter, NULL); - QW_GET_QTID(key, qId, tId, eId); + QW_GET_QTID(key, qId, cId, tId, eId); qwFreeTaskCtx(ctx); QW_TASK_DLOG_E("task ctx freed"); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 9b96c1e519..13e1d0e231 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -19,7 +19,7 @@ SQWorkerMgmt gQwMgmt = { }; void qwStopAllTasks(SQWorker *mgmt) { - uint64_t qId, tId, sId; + uint64_t qId, cId, tId, sId; int32_t eId; int64_t rId = 0; int32_t code = TSDB_CODE_SUCCESS; @@ -28,7 +28,7 @@ void qwStopAllTasks(SQWorker *mgmt) { while (pIter) { SQWTaskCtx *ctx = (SQWTaskCtx *)pIter; void *key = taosHashGetKey(pIter, NULL); - QW_GET_QTID(key, qId, tId, eId); + QW_GET_QTID(key, qId, cId, tId, eId); QW_LOCK(QW_WRITE, &ctx->lock); @@ -288,7 +288,7 @@ int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) // TODO GET EXECUTOR API TO GET MORE INFO - QW_GET_QTID(key, status.queryId, status.taskId, status.execId); + QW_GET_QTID(key, status.queryId, status.clientId, status.taskId, status.execId); status.status = taskStatus->status; status.refId = taskStatus->refId; @@ -1473,8 +1473,8 @@ int32_t qWorkerGetStat(SReadHandle *handle, void *qWorkerMgmt, SQWorkerStat *pSt return TSDB_CODE_SUCCESS; } -int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId, - SQWMsg *qwMsg, SArray *explainRes) { +int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t cId, uint64_t tId, int64_t rId, + int32_t eId, SQWMsg *qwMsg, SArray *explainRes) { SQWorker *mgmt = (SQWorker *)pMgmt; int32_t code = 0; SQWTaskCtx *ctx = NULL; @@ -1538,8 +1538,8 @@ _return: QW_RET(code); } -int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId, - void **pRsp, SArray *explainRes) { +int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t cId, uint64_t tId, int64_t rId, + int32_t eId, void **pRsp, SArray *explainRes) { SQWorker *mgmt = (SQWorker *)pMgmt; int32_t code = 0; int32_t dataLen = 0; diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 96b9d2da8d..6a910453f0 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -142,8 +142,9 @@ typedef struct SSchedulerCfg { } SSchedulerCfg; typedef struct SSchedulerMgmt { - uint64_t taskId; // sequential taksId - uint64_t sId; // schedulerId + uint64_t clientId; // unique clientId + uint64_t taskId; // sequential taksId + uint64_t sId; // schedulerId SSchedulerCfg cfg; bool exit; int32_t jobRef; @@ -163,6 +164,7 @@ typedef struct SSchTaskCallbackParam { SSchCallbackParamHeader head; uint64_t queryId; int64_t refId; + uint64_t clientId; uint64_t taskId; int32_t execId; void *pTrans; @@ -222,6 +224,7 @@ typedef struct SSchTimerParam { } SSchTimerParam; typedef struct SSchTask { + uint64_t clientId; // current client id uint64_t taskId; // task id SRWLatch lock; // task reentrant lock int32_t maxExecTimes; // task max exec times @@ -329,6 +332,7 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOCK_TASK(_task) SCH_LOCK(SCH_WRITE, &(_task)->lock) #define SCH_UNLOCK_TASK(_task) SCH_UNLOCK(SCH_WRITE, &(_task)->lock) +#define SCH_CLIENT_ID(_task) ((_task) ? (_task)->clientId : -1) #define SCH_TASK_ID(_task) ((_task) ? (_task)->taskId : -1) #define SCH_TASK_EID(_task) ((_task) ? (_task)->execId : -1) @@ -449,21 +453,21 @@ extern SSchedulerMgmt schMgmt; #define SCH_JOB_ELOG(param, ...) qError("qid:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) #define SCH_JOB_DLOG(param, ...) qDebug("qid:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) -#define SCH_TASK_ELOG(param, ...) \ - qError("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ - __VA_ARGS__) -#define SCH_TASK_DLOG(param, ...) \ - qDebug("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ - __VA_ARGS__) -#define SCH_TASK_TLOG(param, ...) \ - qTrace("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ - __VA_ARGS__) -#define SCH_TASK_DLOGL(param, ...) \ - qDebugL("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ - __VA_ARGS__) -#define SCH_TASK_WLOG(param, ...) \ - qWarn("qid:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ - __VA_ARGS__) +#define SCH_TASK_ELOG(param, ...) \ + qError("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \ + SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__) +#define SCH_TASK_DLOG(param, ...) \ + qDebug("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \ + SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__) +#define SCH_TASK_TLOG(param, ...) \ + qTrace("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \ + SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__) +#define SCH_TASK_DLOGL(param, ...) \ + qDebugL("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \ + SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__) +#define SCH_TASK_WLOG(param, ...) \ + qWarn("qid:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_CLIENT_ID(pTask), \ + SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), __VA_ARGS__) #define SCH_SET_ERRNO(_err) \ do { \ diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index b15a6a09d3..eefb32f783 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -500,8 +500,8 @@ _return: int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, - code); + qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, + pParam->clientId, pParam->taskId, code); // called if drop task rsp received code (void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error @@ -517,8 +517,8 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { int32_t schHandleNotifyCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", pParam->queryId, pParam->taskId, - code); + qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", pParam->queryId, + pParam->clientId, pParam->taskId, code); if (pMsg) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); @@ -595,6 +595,7 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo param->queryId = pJob->queryId; param->refId = pJob->refId; + param->clientId = SCH_CLIENT_ID(pTask); param->taskId = SCH_TASK_ID(pTask); param->pTrans = pJob->conn.pTrans; param->execId = pTask->execId; @@ -1138,6 +1139,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, req.header.vgId = addr->nodeId; req.sId = schMgmt.sId; req.queryId = pJob->queryId; + req.clientId = pTask->clientId; req.taskId = pTask->taskId; req.phyLen = pTask->msgLen; req.sqlLen = strlen(pJob->sql); @@ -1171,6 +1173,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, qMsg.header.contLen = 0; qMsg.sId = schMgmt.sId; qMsg.queryId = pJob->queryId; + qMsg.clientId = pTask->clientId; qMsg.taskId = pTask->taskId; qMsg.refId = pJob->refId; qMsg.execId = pTask->execId; @@ -1226,6 +1229,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, req.header.vgId = addr->nodeId; req.sId = schMgmt.sId; req.queryId = pJob->queryId; + req.clientId = pTask->clientId; req.taskId = pTask->taskId; req.execId = pTask->execId; @@ -1253,6 +1257,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, qMsg.header.contLen = 0; qMsg.sId = schMgmt.sId; qMsg.queryId = pJob->queryId; + qMsg.clientId = pTask->clientId; qMsg.taskId = pTask->taskId; qMsg.refId = pJob->refId; qMsg.execId = *(int32_t*)param; @@ -1310,6 +1315,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, qMsg.header.contLen = 0; qMsg.sId = schMgmt.sId; qMsg.queryId = pJob->queryId; + qMsg.clientId = pTask->clientId; qMsg.taskId = pTask->taskId; qMsg.refId = pJob->refId; qMsg.execId = pTask->execId; diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index fe24633c12..9be0e3fc40 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -66,6 +66,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * pTask->execId = -1; pTask->failedExecId = -2; pTask->timeoutUsec = SCH_DEFAULT_TASK_TIMEOUT_USEC; + pTask->clientId = getClientId(); pTask->taskId = schGenTaskId(); schInitTaskRetryTimes(pJob, pTask, pLevel); @@ -305,6 +306,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_LOCK(SCH_WRITE, &parent->planLock); SDownstreamSourceNode source = { .type = QUERY_NODE_DOWNSTREAM_SOURCE, + .clientId = pTask->clientId, .taskId = pTask->taskId, .schedId = schMgmt.sId, .execId = pTask->execId, @@ -996,8 +998,8 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) { int32_t code = 0; - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, pStatus->taskId, - pStatus->execId, jobTaskStatusStr(pStatus->status)); + qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, + pStatus->clientId, pStatus->taskId, pStatus->execId, jobTaskStatusStr(pStatus->status)); if (schProcessOnCbBegin(&pJob, &pTask, pStatus->queryId, pStatus->refId, pStatus->taskId)) { continue; @@ -1043,13 +1045,14 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { continue; } - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId); + qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", + localRsp->qId, localRsp->cId, localRsp->tId); pJob = NULL; (void)schAcquireJob(localRsp->rId, &pJob); if (NULL == pJob) { - qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, - localRsp->tId, localRsp->rId); + qWarn("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, + localRsp->qId, localRsp->cId, localRsp->tId, localRsp->rId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_NOT_EXIST); } @@ -1068,8 +1071,8 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { (void)schReleaseJob(pJob->refId); - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, - localRsp->tId, code); + qDebug("QID:0x%" PRIx64 ",CID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", + localRsp->qId, localRsp->cId, localRsp->tId, code); SCH_ERR_JRET(code); @@ -1147,8 +1150,8 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { } } - SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, - pTask->execId, &qwMsg, explainRes)); + SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->clientId, pTask->taskId, + pJob->refId, pTask->execId, &qwMsg, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); @@ -1407,8 +1410,8 @@ int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { } } - SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, - pTask->execId, &pRsp, explainRes)); + SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->clientId, pTask->taskId, + pJob->refId, pTask->execId, &pRsp, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 4697de6f28..ac34099417 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -293,6 +293,18 @@ void schCloseJobRef(void) { } } +int32_t initClientId(void) { + int32_t code = taosGetSystemUUIDU64(&schMgmt.clientId); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to generate clientId since %s", tstrerror(code)); + SCH_ERR_RET(code); + } + qInfo("initialize"); + return TSDB_CODE_SUCCESS; +} + +uint64_t getClientId(void) { return schMgmt.clientId; } + uint64_t schGenTaskId(void) { return atomic_add_fetch_64(&schMgmt.taskId, 1); } #ifdef BUILD_NO_CALL diff --git a/tests/script/api/sameReqidTest.c b/tests/script/api/sameReqidTest.c new file mode 100644 index 0000000000..7507619886 --- /dev/null +++ b/tests/script/api/sameReqidTest.c @@ -0,0 +1,406 @@ +// sample code to verify multiple queries with the same reqid +// to compile: gcc -o sameReqdiTest sameReqidTest.c -ltaos + +#include +#include +#include +#include +#include "taos.h" + +#define NUM_ROUNDS 10 +#define CONST_REQ_ID 12345 +#define TEST_DB "test" + +#define CHECK_CONDITION(condition, prompt, errstr) \ + do { \ + if (!(condition)) { \ + printf("\033[31m[%s:%d] failed to " prompt ", reason: %s\033[0m\n", __func__, __LINE__, errstr); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#define CHECK_RES(res, prompt) CHECK_CONDITION(taos_errno(res) == 0, prompt, taos_errstr(res)) +#define CHECK_CODE(code, prompt) CHECK_CONDITION(code == 0, prompt, taos_errstr(NULL)) + +static TAOS* getNewConnection() { + const char* host = "127.0.0.1"; + const char* user = "root"; + const char* passwd = "taosdata"; + TAOS* taos = NULL; + + taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); + taos = taos_connect(host, user, passwd, "", 0); + CHECK_CONDITION(taos != NULL, "connect to db", taos_errstr(NULL)); + return taos; +} + +static void prepareData(TAOS* taos) { + TAOS_RES* res = NULL; + int32_t code = 0; + + res = taos_query(taos, "create database if not exists " TEST_DB " precision 'ns'"); + CHECK_RES(res, "create database"); + taos_free_result(res); + usleep(100000); + + code = taos_select_db(taos, TEST_DB); + CHECK_CODE(code, "switch to database"); + + res = taos_query(taos, "create table if not exists meters(ts timestamp, a int) tags(area int)"); + CHECK_RES(res, "create stable meters"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t0 using meters tags(0)"); + CHECK_RES(res, "create table t0"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t1 using meters tags(1)"); + CHECK_RES(res, "create table t1"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t2 using meters tags(2)"); + CHECK_RES(res, "create table t2"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t3 using meters tags(3)"); + CHECK_RES(res, "create table t3"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t4 using meters tags(4)"); + CHECK_RES(res, "create table t4"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t5 using meters tags(5)"); + CHECK_RES(res, "create table t5"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t6 using meters tags(6)"); + CHECK_RES(res, "create table t6"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t7 using meters tags(7)"); + CHECK_RES(res, "create table t7"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t8 using meters tags(8)"); + CHECK_RES(res, "create table t8"); + taos_free_result(res); + + res = taos_query(taos, "create table if not exists t9 using meters tags(9)"); + CHECK_RES(res, "create table t9"); + taos_free_result(res); + + res = taos_query(taos, + "insert into t0 values('2020-01-01 00:00:00.000', 0)" + " ('2020-01-01 00:01:00.000', 0)" + " ('2020-01-01 00:02:00.000', 0)" + " t1 values('2020-01-01 00:00:00.000', 1)" + " ('2020-01-01 00:01:00.000', 1)" + " ('2020-01-01 00:02:00.000', 1)" + " ('2020-01-01 00:03:00.000', 1)" + " t2 values('2020-01-01 00:00:00.000', 2)" + " ('2020-01-01 00:01:00.000', 2)" + " ('2020-01-01 00:01:01.000', 2)" + " ('2020-01-01 00:01:02.000', 2)" + " t3 values('2020-01-01 00:01:02.000', 3)" + " t4 values('2020-01-01 00:01:02.000', 4)" + " t5 values('2020-01-01 00:01:02.000', 5)" + " t6 values('2020-01-01 00:01:02.000', 6)" + " t7 values('2020-01-01 00:01:02.000', 7)" + " t8 values('2020-01-01 00:01:02.000', 8)" + " t9 values('2020-01-01 00:01:02.000', 9)"); + CHECK_RES(res, "insert into meters"); + CHECK_CONDITION(taos_affected_rows(res), "insert into meters", "insufficient count"); + taos_free_result(res); + + res = taos_query( + taos, + "create table if not exists m1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 " + "double, bin binary(40), blob nchar(10))"); + CHECK_RES(res, "create table m1"); + taos_free_result(res); + + usleep(1000000); +} + +static void verifySchemaLess(TAOS* taos) { + TAOS_RES* res = NULL; + char* lines[] = { + "st,t1=3i64,t2=4f64,t3=L\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000", + "st,t1=4i64,t3=L\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64 1626006833640000000", + "st,t2=5f64,t3=L\"ste\" c1=4i64,c2=true,c3=L\"iam\" 1626056811823316532", + "st,t1=4i64,t2=5f64,t3=L\"t4\" c1=3i64,c3=L\"passitagain\",c2=true,c4=5f64 1626006833642000000", + "st,t2=5f64,t3=L\"ste2\" c3=L\"iamszhou\",c2=false 1626056811843316532", + "st,t2=5f64,t3=L\"ste2\" c3=L\"iamszhou\",c2=false,c5=5f64,c6=7u64,c7=32i32,c8=88.88f32 1626056812843316532", + "st,t1=4i64,t3=L\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64,c6=7u64 " + "1626006933640000000", + "st,t1=4i64,t3=L\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64,c6=7u64 " + "1626006933640000000", + "st,t1=4i64,t3=L\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 " + "1626006933641000000"}; + + res = taos_schemaless_insert_with_reqid(taos, lines, sizeof(lines) / sizeof(char*), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS, CONST_REQ_ID); + CHECK_RES(res, "insert schema-less data"); + printf("successfully inserted %d rows\n", taos_affected_rows(res)); + taos_free_result(res); +} + +static int32_t printResult(TAOS_RES* res, int32_t nlimit) { + TAOS_ROW row = NULL; + TAOS_FIELD* fields = NULL; + int32_t numFields = 0; + int32_t nRows = 0; + + numFields = taos_num_fields(res); + fields = taos_fetch_fields(res); + while ((nlimit-- > 0) && (row = taos_fetch_row(res))) { + char temp[256] = {0}; + taos_print_row(temp, row, fields, numFields); + puts(temp); + nRows++; + } + return nRows; +} + +static void verifyQuery(TAOS* taos) { + TAOS_RES* res = NULL; + + res = taos_query_with_reqid(taos, "select * from meters", CONST_REQ_ID); + CHECK_RES(res, "select from meters"); + printResult(res, INT32_MAX); + taos_free_result(res); + + res = taos_query_with_reqid(taos, "select * from t0", CONST_REQ_ID); + CHECK_RES(res, "select from t0"); + printResult(res, INT32_MAX); + taos_free_result(res); + + res = taos_query_with_reqid(taos, "select * from t1", CONST_REQ_ID); + CHECK_RES(res, "select from t1"); + printResult(res, INT32_MAX); + taos_free_result(res); + + res = taos_query_with_reqid(taos, "select * from t2", CONST_REQ_ID); + CHECK_RES(res, "select from t2"); + printResult(res, INT32_MAX); + taos_free_result(res); + + res = taos_query_with_reqid(taos, "select * from t3", CONST_REQ_ID); + CHECK_RES(res, "select from t3"); + printResult(res, INT32_MAX); + taos_free_result(res); + + printf("succeed to read from meters\n"); +} + +void retrieveCallback(void* param, TAOS_RES* res, int32_t nrows) { + if (nrows == 0) { + taos_free_result(res); + } else { + printResult(res, nrows); + taos_fetch_rows_a(res, retrieveCallback, param); + } +} + +void selectCallback(void* param, TAOS_RES* res, int32_t code) { + CHECK_CODE(code, "read async from table"); + taos_fetch_rows_a(res, retrieveCallback, param); +} + +static void verifyQueryAsync(TAOS* taos) { + taos_query_a_with_reqid(taos, "select *from meters", selectCallback, NULL, CONST_REQ_ID); + taos_query_a_with_reqid(taos, "select *from t0", selectCallback, NULL, CONST_REQ_ID); + taos_query_a_with_reqid(taos, "select *from t1", selectCallback, NULL, CONST_REQ_ID); + taos_query_a_with_reqid(taos, "select *from t2", selectCallback, NULL, CONST_REQ_ID); + taos_query_a_with_reqid(taos, "select *from t3", selectCallback, NULL, CONST_REQ_ID); + + sleep(1); +} + +void veriryStmt(TAOS* taos) { + // insert 10 records + struct { + int64_t ts[10]; + int8_t b[10]; + int8_t v1[10]; + int16_t v2[10]; + int32_t v4[10]; + int64_t v8[10]; + float f4[10]; + double f8[10]; + char bin[10][40]; + char blob[10][80]; + } v; + + int32_t* t8_len = malloc(sizeof(int32_t) * 10); + int32_t* t16_len = malloc(sizeof(int32_t) * 10); + int32_t* t32_len = malloc(sizeof(int32_t) * 10); + int32_t* t64_len = malloc(sizeof(int32_t) * 10); + int32_t* float_len = malloc(sizeof(int32_t) * 10); + int32_t* double_len = malloc(sizeof(int32_t) * 10); + int32_t* bin_len = malloc(sizeof(int32_t) * 10); + int32_t* blob_len = malloc(sizeof(int32_t) * 10); + + TAOS_STMT* stmt = taos_stmt_init_with_reqid(taos, CONST_REQ_ID); + TAOS_MULTI_BIND params[10]; + char is_null[10] = {0}; + + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(v.ts[0]); + params[0].buffer = v.ts; + params[0].length = t64_len; + params[0].is_null = is_null; + params[0].num = 10; + + params[1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[1].buffer_length = sizeof(v.b[0]); + params[1].buffer = v.b; + params[1].length = t8_len; + params[1].is_null = is_null; + params[1].num = 10; + + params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[2].buffer_length = sizeof(v.v1[0]); + params[2].buffer = v.v1; + params[2].length = t8_len; + params[2].is_null = is_null; + params[2].num = 10; + + params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[3].buffer_length = sizeof(v.v2[0]); + params[3].buffer = v.v2; + params[3].length = t16_len; + params[3].is_null = is_null; + params[3].num = 10; + + params[4].buffer_type = TSDB_DATA_TYPE_INT; + params[4].buffer_length = sizeof(v.v4[0]); + params[4].buffer = v.v4; + params[4].length = t32_len; + params[4].is_null = is_null; + params[4].num = 10; + + params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[5].buffer_length = sizeof(v.v8[0]); + params[5].buffer = v.v8; + params[5].length = t64_len; + params[5].is_null = is_null; + params[5].num = 10; + + params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[6].buffer_length = sizeof(v.f4[0]); + params[6].buffer = v.f4; + params[6].length = float_len; + params[6].is_null = is_null; + params[6].num = 10; + + params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[7].buffer_length = sizeof(v.f8[0]); + params[7].buffer = v.f8; + params[7].length = double_len; + params[7].is_null = is_null; + params[7].num = 10; + + params[8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[8].buffer_length = sizeof(v.bin[0]); + params[8].buffer = v.bin; + params[8].length = bin_len; + params[8].is_null = is_null; + params[8].num = 10; + + params[9].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[9].buffer_length = sizeof(v.blob[0]); + params[9].buffer = v.blob; + params[9].length = blob_len; + params[9].is_null = is_null; + params[9].num = 10; + + int32_t code = taos_stmt_prepare( + stmt, "insert into ? (ts, b, v1, v2, v4, v8, f4, f8, bin, blob) values(?,?,?,?,?,?,?,?,?,?)", 0); + CHECK_CODE(code, "taos_stmt_prepare"); + + code = taos_stmt_set_tbname(stmt, "m1"); + CHECK_CODE(code, "taos_stmt_set_tbname"); + + int64_t ts = 1591060628000000000; + for (int i = 0; i < 10; ++i) { + v.ts[i] = ts; + ts += 1000000; + is_null[i] = 0; + + v.b[i] = (int8_t)i % 2; + v.v1[i] = (int8_t)i; + v.v2[i] = (int16_t)(i * 2); + v.v4[i] = (int32_t)(i * 4); + v.v8[i] = (int64_t)(i * 8); + v.f4[i] = (float)(i * 40); + v.f8[i] = (double)(i * 80); + for (int j = 0; j < sizeof(v.bin[0]); ++j) { + v.bin[i][j] = (char)(i + '0'); + } + strcpy(v.blob[i], "一二三四五六七八九十"); + + t8_len[i] = sizeof(int8_t); + t16_len[i] = sizeof(int16_t); + t32_len[i] = sizeof(int32_t); + t64_len[i] = sizeof(int64_t); + float_len[i] = sizeof(float); + double_len[i] = sizeof(double); + bin_len[i] = sizeof(v.bin[0]); + blob_len[i] = (int32_t)strlen(v.blob[i]); + } + + code = taos_stmt_bind_param_batch(stmt, params); + CHECK_CODE(code, "taos_stmt_bind_param_batch"); + + code = taos_stmt_add_batch(stmt); + CHECK_CODE(code, "taos_stmt_add_batch"); + + code = taos_stmt_execute(stmt); + CHECK_CODE(code, "taos_stmt_execute"); + + taos_stmt_close(stmt); + + free(t8_len); + free(t16_len); + free(t32_len); + free(t64_len); + free(float_len); + free(double_len); + free(bin_len); + free(blob_len); +} + +int main(int argc, char* argv[]) { + TAOS* taos = NULL; + int32_t code = 0; + + taos = getNewConnection(); + taos_select_db(taos, TEST_DB); + CHECK_CODE(code, "switch to database"); + + printf("************ prepare data *************\n"); + prepareData(taos); + + for (int32_t i = 0; i < NUM_ROUNDS; ++i) { + printf("************ verify schema-less *************\n"); + verifySchemaLess(taos); + + printf("************ verify query *************\n"); + verifyQuery(taos); + + printf("********* verify async query **********\n"); + verifyQueryAsync(taos); + + printf("********* verify stmt query **********\n"); + veriryStmt(taos); + + printf("done\n"); + } + + taos_close(taos); + taos_cleanup(); + + return 0; +} From 25775cc14a9bc61daf463465b0564d006fedcec3 Mon Sep 17 00:00:00 2001 From: qevolg <2227465945@qq.com> Date: Thu, 7 Nov 2024 09:56:06 +0800 Subject: [PATCH 05/44] fix(keeper): upgrading taoskeeper deps --- tools/keeper/go.mod | 31 ++++++++++---------- tools/keeper/go.sum | 71 +++++++++++++++++++++++++-------------------- 2 files changed, 55 insertions(+), 47 deletions(-) diff --git a/tools/keeper/go.mod b/tools/keeper/go.mod index f520ceb774..f8edf2709b 100644 --- a/tools/keeper/go.mod +++ b/tools/keeper/go.mod @@ -21,38 +21,39 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect - github.com/bytedance/sonic v1.9.1 // indirect + github.com/bytedance/sonic v1.11.2 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect + github.com/chenzhuoyu/iasm v0.9.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect - github.com/gabriel-vasile/mimetype v1.4.2 // indirect - github.com/gin-contrib/cors v1.3.1 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/cors v1.6.0 // indirect github.com/gin-contrib/gzip v0.0.3 // indirect github.com/gin-contrib/pprof v1.3.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.14.0 // indirect + github.com/go-playground/validator/v10 v10.19.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect - github.com/leodido/go-urn v1.2.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.6 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -66,14 +67,14 @@ require ( github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.11 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect + golang.org/x/arch v0.7.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/sys v0.24.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/tools/keeper/go.sum b/tools/keeper/go.sum index 9c7721c4d7..8f6e9bd13a 100644 --- a/tools/keeper/go.sum +++ b/tools/keeper/go.sum @@ -52,15 +52,20 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= -github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= +github.com/bytedance/sonic v1.11.2 h1:ywfwo0a/3j9HR8wsYGWsIWl2mvRsI950HyoxiBERw5A= +github.com/bytedance/sonic v1.11.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= +github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= +github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= +github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0= +github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -88,10 +93,11 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= -github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= -github.com/gin-contrib/cors v1.3.1 h1:doAsuITavI4IOcd0Y19U4B+O0dNWihRyX//nn4sEmgA= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gin-contrib/cors v1.3.1/go.mod h1:jjEJ4268OPZUcU7k9Pm653S7lXUGcqMADzFA61xsmDk= +github.com/gin-contrib/cors v1.6.0 h1:0Z7D/bVhE6ja07lI8CTjTonp6SB07o8bNuFyRbsBUQg= +github.com/gin-contrib/cors v1.6.0/go.mod h1:cI+h6iOAyxKRtUtC6iF/Si1KSFvGm/gK+kshxlCi8ro= github.com/gin-contrib/gzip v0.0.3 h1:etUaeesHhEORpZMp18zoOhepboiWnFtXrBZxszWUn4k= github.com/gin-contrib/gzip v0.0.3/go.mod h1:YxxswVZIqOvcHEQpsSn+QF5guQtO1dCfy0shBPy4jFc= github.com/gin-contrib/pprof v1.3.0 h1:G9eK6HnbkSqDZBYbzG4wrjCsA4e+cvYAHUZw6W+W9K0= @@ -127,8 +133,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= -github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= -github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= +github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -235,8 +241,9 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -250,8 +257,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= -github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8= github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= @@ -262,8 +269,8 @@ github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamh github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -290,8 +297,8 @@ github.com/panjf2000/ants/v2 v2.4.6 h1:drmj9mcygn2gawZ155dRbo+NfXEfAssjZNU1qoIb4 github.com/panjf2000/ants/v2 v2.4.6/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= -github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= +github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -330,7 +337,7 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/shirou/gopsutil/v3 v3.22.4 h1:srAQaiX6jX/cYL6q29aE0m8lOskT9CurZ9N61YR3yoI= github.com/shirou/gopsutil/v3 v3.22.4/go.mod h1:D01hZJ4pVHPpCTZ3m3T2+wDF2YAGfd+H4ifUguaQzHM= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= @@ -363,8 +370,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI= @@ -387,8 +393,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/xdg/scram v1.0.3/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -404,8 +410,8 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= -golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= +golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -418,8 +424,8 @@ golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -489,8 +495,8 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -565,7 +571,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -577,8 +583,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -725,8 +731,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -758,6 +764,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= From 5cc02a774f685822eced9d50a9e1761001cc201d Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 7 Nov 2024 11:08:54 +0800 Subject: [PATCH 06/44] test: add smoke test scripts --- .gitignore | 2 +- packaging/smoke_test/conftest.py | 115 ++++++ packaging/smoke_test/debRpmAutoInstall.sh | 15 + packaging/smoke_test/getAndRunInstaller.bat | 57 +++ packaging/smoke_test/getAndRunInstaller.sh | 325 +++++++++++++++ packaging/smoke_test/lib.py | 12 + packaging/smoke_test/main.py | 21 + packaging/smoke_test/pytest_require.txt | 16 + packaging/smoke_test/runCases.bat | 11 + packaging/smoke_test/runCases.sh | 29 ++ packaging/smoke_test/smokeTestClient.py | 251 ++++++++++++ packaging/smoke_test/smokeTestJenkinsFile | 380 ++++++++++++++++++ packaging/smoke_test/start3NodesServer.sh | 67 +++ packaging/smoke_test/test_client.py | 137 +++++++ packaging/smoke_test/test_server.py | 238 +++++++++++ packaging/smoke_test/test_server_unix_case | 10 + packaging/smoke_test/test_server_windows_case | 2 + .../smoke_test/versionCheckAndUninstall.py | 260 ++++++++++++ .../versionCheckAndUninstallforPytest.py | 137 +++++++ tests/system-test/test.py | 5 +- 20 files changed, 2088 insertions(+), 2 deletions(-) create mode 100644 packaging/smoke_test/conftest.py create mode 100755 packaging/smoke_test/debRpmAutoInstall.sh create mode 100644 packaging/smoke_test/getAndRunInstaller.bat create mode 100755 packaging/smoke_test/getAndRunInstaller.sh create mode 100644 packaging/smoke_test/lib.py create mode 100644 packaging/smoke_test/main.py create mode 100644 packaging/smoke_test/pytest_require.txt create mode 100644 packaging/smoke_test/runCases.bat create mode 100644 packaging/smoke_test/runCases.sh create mode 100644 packaging/smoke_test/smokeTestClient.py create mode 100644 packaging/smoke_test/smokeTestJenkinsFile create mode 100644 packaging/smoke_test/start3NodesServer.sh create mode 100644 packaging/smoke_test/test_client.py create mode 100644 packaging/smoke_test/test_server.py create mode 100644 packaging/smoke_test/test_server_unix_case create mode 100644 packaging/smoke_test/test_server_windows_case create mode 100644 packaging/smoke_test/versionCheckAndUninstall.py create mode 100644 packaging/smoke_test/versionCheckAndUninstallforPytest.py diff --git a/.gitignore b/.gitignore index d684262f9d..7f2382db87 100644 --- a/.gitignore +++ b/.gitignore @@ -161,4 +161,4 @@ version.h geos_c.h source/libs/parser/src/sql.c include/common/ttokenauto.h - +!packaging/smoke_test/pytest_require.txt diff --git a/packaging/smoke_test/conftest.py b/packaging/smoke_test/conftest.py new file mode 100644 index 0000000000..a5f6ebbbe9 --- /dev/null +++ b/packaging/smoke_test/conftest.py @@ -0,0 +1,115 @@ +# conftest.py +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--verMode", default="enterprise", help="community or enterprise" + ) + parser.addoption( + "--tVersion", default="3.3.2.6", help="the version of taos" + ) + parser.addoption( + "--baseVersion", default="smoking", help="the path of nas" + ) + parser.addoption( + "--sourcePath", default="nas", help="only support nas currently" + ) + + + + +# Collect the setup and teardown of each test case and their std information +setup_stdout_info = {} +teardown_stdout_info = {} + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + rep = outcome.get_result() + + # Record the std of setup and teardown + if call.when == 'setup': + for i in rep.sections: + if i[0] == "Captured stdout setup": + if not setup_stdout_info: + setup_stdout_info[item.nodeid] = i[1] + elif call.when == 'teardown': + for i in rep.sections: + if i[0] == "Captured stdout teardown": + teardown_stdout_info[item.nodeid] = i[1] + + +# Insert setup and teardown's std in the summary section +def pytest_html_results_summary(prefix, summary, postfix): + if setup_stdout_info or teardown_stdout_info: + rows = [] + + # Insert setup stdout + if setup_stdout_info: + for nodeid, stdout in setup_stdout_info.items(): + html_content = ''' + + Setup: + + Show Setup + + + + '''.format(stdout.strip()) + + # 如果需要在 Python 脚本中生成 HTML，并使用 JavaScript 控制折叠内容的显示，可以这样做： + + html_script = ''' + + ''' + + # 输出完整的 HTML 代码 + final_html = html_content + html_script + rows.append(final_html) + rows.append("
") + # Insert teardown stdout + if teardown_stdout_info: + for nodeid, stdout in teardown_stdout_info.items(): + html_content = ''' + + Teardown: + + Show Teardown + + + + '''.format(stdout.strip()) + + # 如果需要在 Python 脚本中生成 HTML，并使用 JavaScript 控制折叠内容的显示，可以这样做： + + html_script = ''' + + ''' + + # 输出完整的 HTML 代码 + final_html = html_content + html_script + rows.append(final_html) + + prefix.extend(rows) diff --git a/packaging/smoke_test/debRpmAutoInstall.sh b/packaging/smoke_test/debRpmAutoInstall.sh new file mode 100755 index 0000000000..8fadffe4c6 --- /dev/null +++ b/packaging/smoke_test/debRpmAutoInstall.sh @@ -0,0 +1,15 @@ +#!/usr/bin/expect +set packageName [lindex $argv 0] +set packageSuffix [lindex $argv 1] +set timeout 30 +if { ${packageSuffix} == "deb" } { + spawn dpkg -i ${packageName} +} elseif { ${packageSuffix} == "rpm"} { + spawn rpm -ivh ${packageName} +} +expect "*one:" +send "\r" +expect "*skip:" +send "\r" + +expect eof diff --git a/packaging/smoke_test/getAndRunInstaller.bat b/packaging/smoke_test/getAndRunInstaller.bat new file mode 100644 index 0000000000..08b04a0271 --- /dev/null +++ b/packaging/smoke_test/getAndRunInstaller.bat @@ -0,0 +1,57 @@ +set baseVersion=%1% +set version=%2% +set verMode=%3% +set sType=%4% +echo %fileType% +rem stop services +if EXIST C:\TDengine ( + if EXIST C:\TDengine\stop-all.bat ( + call C:\TDengine\stop-all.bat /silent + echo "***************Stop taos services***************" + ) + if exist C:\TDengine\unins000.exe ( + call C:\TDengine\unins000.exe /silent + echo "***************uninstall TDengine***************" + ) + rd /S /q C:\TDengine +) +if EXIST C:\ProDB ( + if EXIST C:\ProDB\stop-all.bat ( + call C:\ProDB\stop-all.bat /silent + echo "***************Stop taos services***************" + ) + if exist C:\ProDB\unins000.exe ( + call C:\ProDB\unins000.exe /silent + echo "***************uninstall TDengine***************" + ) + rd /S /q C:\ProDB +) +if "%verMode%"=="enterprise" ( + if "%sType%"=="client" ( + set fileType=enterprise-client + ) else ( + set fileType=enterprise + ) +) else ( + set fileType=%sType% +) + +if "%baseVersion%"=="ProDB" ( + echo %fileType% + set installer=ProDB-%fileType%-%version%-Windows-x64.exe +) else ( + echo %fileType% + set installer=TDengine-%fileType%-%version%-Windows-x64.exe +) + +if "%baseVersion%"=="ProDB" ( + echo %installer% + scp root@192.168.1.213:/nas/OEM/ProDB/v%version%/%installer% C:\workspace +) else ( + echo %installer% + scp root@192.168.1.213:/nas/TDengine/%baseVersion%/v%version%/%verMode%/%installer% C:\workspace +) + +echo "***************Finish installer transfer!***************" +C:\workspace\%installer% /silent +echo "***************Finish install!***************" \ No newline at end of file diff --git a/packaging/smoke_test/getAndRunInstaller.sh b/packaging/smoke_test/getAndRunInstaller.sh new file mode 100755 index 0000000000..7defe6394c --- /dev/null +++ b/packaging/smoke_test/getAndRunInstaller.sh @@ -0,0 +1,325 @@ +#!/bin/sh + + +function usage() { + echo "$0" + echo -e "\t -f test file type,server/client/tools/" + echo -e "\t -m pacakage version Type,community/enterprise" + echo -e "\t -l package type,lite or not" + echo -e "\t -c operation type,x64/arm64" + echo -e "\t -v pacakage version,3.0.1.7" + echo -e "\t -o pacakage version,3.0.1.7" + echo -e "\t -s source Path,web/nas" + echo -e "\t -t package Type,tar/rpm/deb" + echo -e "\t -h help" +} + + +#parameter +scriptDir=$(dirname $(readlink -f $0)) +version="3.0.1.7" +originversion="smoking" +testFile="server" +verMode="communtity" +sourcePath="nas" +cpuType="x64" +lite="true" +packageType="tar" +subFile="package.tar.gz" +while getopts "m:c:f:l:s:o:t:v:h" opt; do + case $opt in + m) + verMode=$OPTARG + ;; + v) + version=$OPTARG + ;; + f) + testFile=$OPTARG + ;; + l) + lite=$OPTARG + ;; + s) + sourcePath=$OPTARG + ;; + o) + originversion=$OPTARG + ;; + c) + cpuType=$OPTARG + ;; + t) + packageType=$OPTARG + ;; + h) + usage + exit 0 + ;; + ?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; + esac +done + +systemType=`uname` +if [ ${systemType} == "Darwin" ]; then + platform="macOS" +else + platform="Linux" +fi + +echo "testFile:${testFile},verMode:${verMode},lite:${lite},cpuType:${cpuType},packageType:${packageType},version-${version},originversion:${originversion},sourcePath:${sourcePath}" +# Color setting +RED='\033[41;30m' +GREEN='\033[1;32m' +YELLOW='\033[1;33m' +BLUE='\033[1;34m' +GREEN_DARK='\033[0;32m' +YELLOW_DARK='\033[0;33m' +BLUE_DARK='\033[0;34m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' +if [ "${originversion}" = "ProDB" ]; then + TDengine="ProDB" +else + TDengine="TDengine" +fi +if [[ ${verMode} = "enterprise" ]];then + prePackage="${TDengine}-enterprise" + if [[ ${testFile} = "client" ]];then + prePackage="${TDengine}-enterprise-${testFile}" + fi +elif [ ${verMode} = "community" ];then + prePackage="${TDengine}-${testFile}" +fi +if [ ${lite} = "true" ];then + packageLite="-Lite" +elif [ ${lite} = "false" ];then + packageLite="" +fi +if [[ "$packageType" = "tar" ]] ;then + packageType="tar.gz" +fi + +tdPath="${prePackage}-${version}" + +packageName="${tdPath}-${platform}-${cpuType}${packageLite}.${packageType}" + +if [ "$testFile" == "server" ] ;then + installCmd="install.sh" +elif [ ${testFile} = "client" ];then + installCmd="install_client.sh" +fi + +echo "tdPath:${tdPath},packageName:${packageName}}" +cmdInstall() { +command=$1 +if command -v ${command} ;then + echoColor YD "${command} is already installed" +else + if command -v apt ;then + apt-get install ${command} -y + elif command -v yum ;then + yum -y install ${command} + echoColor YD "you should install ${command} manually" + fi +fi +} + +echoColor() { + color=$1 + command=$2 + if [ ${color} = 'Y' ];then + echo -e "${YELLOW}${command}${NC}" + elif [ ${color} = 'YD' ];then + echo -e "${YELLOW_DARK}${command}${NC}" + elif [ ${color} = 'R' ];then + echo -e "${RED}${command}${NC}" + elif [ ${color} = 'G' ];then + echo -e "${GREEN}${command}${NC}\r\n" + elif [ ${color} = 'B' ];then + echo -e "${BLUE}${command}${NC}" + elif [ ${color} = 'BD' ];then + echo -e "${BLUE_DARK}${command}${NC}" + fi +} + +wgetFile() { + + file=$1 + versionPath=$2 + sourceP=$3 + nasServerIP="192.168.1.213" + if [ "${originversion}" = "ProDB" ]; then + packagePath="/nas/OEM/ProDB/v${versionPath}" + else + packagePath="/nas/TDengine/${originversion}/v${versionPath}/${verMode}" + fi + if [ -f ${file} ];then + echoColor YD "${file} already exists ,it will delete it and download it again " + # rm -rf ${file} + fi + + if [[ ${sourceP} = 'web' ]];then + echoColor BD "====download====:wget https://www.taosdata.com/assets-download/3.0/${file}" + wget https://www.taosdata.com/assets-download/3.0/${file} + elif [[ ${sourceP} = 'nas' ]];then + echoColor BD "====download====:scp root@${nasServerIP}:${packagePath}/${file} ." + scp root@${nasServerIP}:${packagePath}/${file} . + fi +} + +function newPath { + +buildPath=$1 + +if [ ! -d ${buildPath} ] ;then + echoColor BD "mkdir -p ${buildPath}" + mkdir -p ${buildPath} +else + echoColor YD "${buildPath} already exists" +fi + +} + +echoColor G "===== install basesoft =====" +cmdInstall tree +cmdInstall wget +cmdInstall expect + +echoColor G "===== Uninstall all components of TDeingne =====" + +if command -v rmtaos ;then + echoColor YD "uninstall all components of TDeingne:rmtaos" + rmtaos +else + echoColor YD "os doesn't include TDengine" +fi + +if [[ ${packageName} =~ "server" ]] ;then + echoColor BD " pkill -9 taosd " + pkill -9 taosd +fi + +if command -v rmprodb ;then + echoColor YD "uninstall all components of TDeingne:rmprodb" + rmprodb +else + echoColor YD "os doesn't include TDengine" +fi + +if [[ ${packageName} =~ "server" ]] ;then + echoColor BD " pkill -9 prodbd " + pkill -9 prodbd +fi + +echoColor G "===== new workroom path =====" +installPath="/usr/local/src/packageTest" + +if [ ${systemType} == "Darwin" ]; then + installPath="${WORK_DIR}/packageTest" +fi + +newPath ${installPath} + +#if [ -d ${installPath}/${tdPath} ] ;then +# echoColor BD "rm -rf ${installPath}/${tdPath}/*" +# rm -rf ${installPath}/${tdPath}/* +#fi + +echoColor G "===== download installPackage =====" +cd ${installPath} && wgetFile ${packageName} ${version} ${sourcePath} +#cd ${oriInstallPath} && wgetFile ${originPackageName} ${originversion} ${sourcePath} + + +cd ${installPath} +cp -r ${scriptDir}/debRpmAutoInstall.sh . + +packageSuffix=$(echo ${packageName} | awk -F '.' '{print $NF}') + + +if [ ! -f debRpmAutoInstall.sh ];then + echo '#!/usr/bin/expect ' > debRpmAutoInstall.sh + echo 'set packageName [lindex $argv 0]' >> debRpmAutoInstall.sh + echo 'set packageSuffix [lindex $argv 1]' >> debRpmAutoInstall.sh + echo 'set timeout 30 ' >> debRpmAutoInstall.sh + echo 'if { ${packageSuffix} == "deb" } {' >> debRpmAutoInstall.sh + echo ' spawn dpkg -i ${packageName} ' >> debRpmAutoInstall.sh + echo '} elseif { ${packageSuffix} == "rpm"} {' >> debRpmAutoInstall.sh + echo ' spawn rpm -ivh ${packageName}' >> debRpmAutoInstall.sh + echo '}' >> debRpmAutoInstall.sh + echo 'expect "*one:"' >> debRpmAutoInstall.sh + echo 'send "\r"' >> debRpmAutoInstall.sh + echo 'expect "*skip:"' >> debRpmAutoInstall.sh + echo 'send "\r" ' >> debRpmAutoInstall.sh +fi + + +echoColor G "===== install Package =====" + +if [[ ${packageName} =~ "deb" ]];then + cd ${installPath} + dpkg -r taostools + dpkg -r tdengine + if [[ ${packageName} =~ "TDengine" ]];then + echoColor BD "./debRpmAutoInstall.sh ${packageName} ${packageSuffix}" && chmod 755 debRpmAutoInstall.sh && ./debRpmAutoInstall.sh ${packageName} ${packageSuffix} + else + echoColor BD "dpkg -i ${packageName}" && dpkg -i ${packageName} + fi +elif [[ ${packageName} =~ "rpm" ]];then + cd ${installPath} + sudo rpm -e tdengine + sudo rpm -e taostools + if [[ ${packageName} =~ "TDengine" ]];then + echoColor BD "./debRpmAutoInstall.sh ${packageName} ${packageSuffix}" && chmod 755 debRpmAutoInstall.sh && ./debRpmAutoInstall.sh ${packageName} ${packageSuffix} + else + echoColor BD "rpm -ivh ${packageName}" && rpm -ivh ${packageName} + fi +elif [[ ${packageName} =~ "tar" ]];then + echoColor G "===== check installPackage File of tar =====" + + cd ${installPath} + echoColor YD "unzip the new installation package" + echoColor BD "tar -xf ${packageName}" && tar -xf ${packageName} + + cd ${installPath}/${tdPath} && tree -I "driver" > ${installPath}/now_${version}_checkfile + + cd ${installPath} + diff ${installPath}/base_${originversion}_checkfile ${installPath}/now_${version}_checkfile > ${installPath}/diffFile.log + diffNumbers=`cat ${installPath}/diffFile.log |wc -l ` + + if [ ${diffNumbers} != 0 ];then + echoColor R "The number and names of files is different from the previous installation package" + diffLog=`cat ${installPath}/diffFile.log` + echoColor Y "${diffLog}" + exit -1 + else + echoColor G "The number and names of files are the same as previous installation packages" + rm -rf ${installPath}/diffFile.log + fi + echoColor YD "===== install Package of tar =====" + cd ${installPath}/${tdPath} + if [ ${testFile} = "server" ];then + echoColor BD "bash ${installCmd} -e no " + bash ${installCmd} -e no + else + echoColor BD "bash ${installCmd} " + bash ${installCmd} + fi +elif [[ ${packageName} =~ "pkg" ]];then + cd ${installPath} + sudo installer -pkg ${packageName} -target / + echoColor YD "===== install Package successfully! =====" +fi + +#cd ${installPath} +# +#rm -rf ${installPath}/${packageName} +#if [ ${platform} == "Linux" ]; then +# rm -rf ${installPath}/${tdPath}/ +#fi +echoColor YD "===== end of shell file =====" + diff --git a/packaging/smoke_test/lib.py b/packaging/smoke_test/lib.py new file mode 100644 index 0000000000..86c30bf8b1 --- /dev/null +++ b/packaging/smoke_test/lib.py @@ -0,0 +1,12 @@ + +import subprocess + + +def run_cmd(command): + print("CMD:", command) + result = subprocess.run(command, capture_output=True, text=True, shell=True) + print("STDOUT:", result.stdout) + print("STDERR:", result.stderr) + print("Return Code:", result.returncode) + #assert result.returncode == 0 + return result diff --git a/packaging/smoke_test/main.py b/packaging/smoke_test/main.py new file mode 100644 index 0000000000..cb7356f80e --- /dev/null +++ b/packaging/smoke_test/main.py @@ -0,0 +1,21 @@ +import pytest + +# python3 -m pytest test_server.py -v --html=/var/www/html/report.html --json-report --json-report-file="/var/www/html/report.json" --timeout=60 + +# pytest.main(["-s", "-v"]) +import pytest + +import subprocess + + +# define cmd function + + + + +def main(): + pytest.main(['--html=report.html']) + + +if __name__ == '__main__': + main() diff --git a/packaging/smoke_test/pytest_require.txt b/packaging/smoke_test/pytest_require.txt new file mode 100644 index 0000000000..eadbe92470 --- /dev/null +++ b/packaging/smoke_test/pytest_require.txt @@ -0,0 +1,16 @@ +pytest-html +pytest-json-report +pytest-timeout +taospy +numpy +fabric2 +psutil +pandas +toml +distro +requests +pexpect +faker +pyopenssl +taos-ws-py +taospy \ No newline at end of file diff --git a/packaging/smoke_test/runCases.bat b/packaging/smoke_test/runCases.bat new file mode 100644 index 0000000000..922766785c --- /dev/null +++ b/packaging/smoke_test/runCases.bat @@ -0,0 +1,11 @@ +rm -rf %WIN_TDENGINE_ROOT_DIR%\debug +mkdir %WIN_TDENGINE_ROOT_DIR%\debug +mkdir %WIN_TDENGINE_ROOT_DIR%\debug\build +mkdir %WIN_TDENGINE_ROOT_DIR%\debug\build\bin +xcopy C:\TDengine\taos*.exe %WIN_TDENGINE_ROOT_DIR%\debug\build\bin + +set case_out_file=%cd%\case.out + +cd %WIN_TDENGINE_ROOT_DIR%\tests\system-test +python3 .\test.py -f 0-others\taosShell.py +python3 .\test.py -f 6-cluster\5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py -N 6 -M 3 \ No newline at end of file diff --git a/packaging/smoke_test/runCases.sh b/packaging/smoke_test/runCases.sh new file mode 100644 index 0000000000..4de7a7658b --- /dev/null +++ b/packaging/smoke_test/runCases.sh @@ -0,0 +1,29 @@ +#!/bin/bash +ulimit -c unlimited + +rm -rf ${TDENGINE_ROOT_DIR}/debug +mkdir ${TDENGINE_ROOT_DIR}/debug +mkdir ${TDENGINE_ROOT_DIR}/debug/build +mkdir ${TDENGINE_ROOT_DIR}/debug/build/bin + +systemType=`uname` +if [ ${systemType} == "Darwin" ]; then + cp /usr/local/bin/taos* ${TDENGINE_ROOT_DIR}/debug/build/bin/ +else + cp /usr/bin/taos* ${TDENGINE_ROOT_DIR}/debug/build/bin/ +fi + +case_out_file=`pwd`/case.out +python3 -m pip install -r ${TDENGINE_ROOT_DIR}/tests/requirements.txt >> $case_out_file +python3 -m pip install taos-ws-py taospy >> $case_out_file + +cd ${TDENGINE_ROOT_DIR}/tests/army +python3 ./test.py -f query/query_basic.py -N 3 >> $case_out_file + +cd ${TDENGINE_ROOT_DIR}/tests/system-test +python3 ./test.py -f 1-insert/insert_column_value.py >> $case_out_file +python3 ./test.py -f 2-query/primary_ts_base_5.py >> $case_out_file +python3 ./test.py -f 2-query/case_when.py >> $case_out_file +python3 ./test.py -f 2-query/partition_limit_interval.py >> $case_out_file +python3 ./test.py -f 2-query/join.py >> $case_out_file +python3 ./test.py -f 2-query/fill.py >> $case_out_file diff --git a/packaging/smoke_test/smokeTestClient.py b/packaging/smoke_test/smokeTestClient.py new file mode 100644 index 0000000000..eee9667300 --- /dev/null +++ b/packaging/smoke_test/smokeTestClient.py @@ -0,0 +1,251 @@ +#!/usr/bin/python +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### +# install pip +# pip install src/connector/python/ + +# -*- coding: utf-8 -*- +import sys, os +import re +import platform +import getopt +import subprocess +# from this import d +import time + +# input for server + +opts, args = getopt.gnu_getopt(sys.argv[1:], 'h:P:v:u', [ + 'host=', 'Port=', 'version=']) +serverHost = "" +serverPort = 0 +version = "" +uninstall = False +for key, value in opts: + if key in ['--help']: + print('A collection of test cases written using Python') + print('-h serverHost') + print('-P serverPort') + print('-v test client version') + print('-u test uninstall process, will uninstall TDengine') + sys.exit(0) + + if key in ['-h']: + serverHost = value + if key in ['-P']: + serverPort = int(value) + if key in ['-v']: + version = value + if key in ['-u']: + uninstall = True +if not serverHost: + print("Please input use -h to specify your server host.") + sys.exit(0) +if not version: + print("No version specified, will not run version check.") +if serverPort == 0: + serverPort = 6030 + print("No server port specified, use default 6030.") + + +system = platform.system() + +arch = platform.machine() + +databaseName = re.sub(r'[^a-zA-Z0-9]', '', subprocess.getoutput("hostname")).lower() +# install taospy +taospy_version = "" +if system == 'Windows': + taospy_version = subprocess.getoutput("pip3 show taospy|findstr Version") +else: + taospy_version = subprocess.getoutput("pip3 show taospy|grep Version| awk -F ':' '{print $2}' ") + +print("taospy version %s " % taospy_version) +if taospy_version == "": + subprocess.getoutput("pip3 install git+https://github.com/taosdata/taos-connector-python.git") + print("install taos python connector") +else: + subprocess.getoutput("pip3 install taospy") + +# prepare data by taosBenchmark +cmd = "taosBenchmark -y -a 3 -n 100 -t 100 -d %s -h %s -P %d &" % (databaseName, serverHost, serverPort) +process_out = subprocess.getoutput(cmd) +print(cmd) +#os.system("taosBenchmark -y -a 3 -n 100 -t 100 -d %s -h %s -P %d" % (databaseName, serverHost, serverPort)) +taosBenchmark_test_result = True +time.sleep(10) +import taos + +conn = taos.connect(host=serverHost, + user="root", + password="taosdata", + database=databaseName, + port=serverPort, + timezone="Asia/Shanghai") # default your host's timezone + +server_version = conn.server_info +print("server_version", server_version) +client_version = conn.client_info +print("client_version", client_version) # 3.0.0.0 + +# Execute a sql and get its result set. It's useful for SELECT statement +result: taos.TaosResult = conn.query("SELECT count(*) from meters") + +data = result.fetch_all() +print(data) +if data[0][0] !=10000: + print(" taosBenchmark work not as expected ") + print("!!!!!!!!!!!Test Result: taosBenchmark test failed! !!!!!!!!!!") + sys.exit(1) +#else: +# print("**********Test Result: taosBenchmark test passed **********") + + +# drop database of test +taos_test_result = False +print("drop database test") +print("run taos -s 'drop database %s;' -h %s -P %d" % (databaseName, serverHost, serverPort)) +taos_cmd_outpur = subprocess.getoutput('taos -s "drop database %s;" -h %s -P %d' % (databaseName, serverHost, serverPort)) +print(taos_cmd_outpur) +if ("Drop OK" in taos_cmd_outpur): + taos_test_result = True + #print("*******Test Result: taos test passed ************") + +version_test_result = False +if version: + print("Client info is: %s"%conn.client_info) + taos_V_output = "" + if system == "Windows": + taos_V_output = subprocess.getoutput("taos -V | findstr version") + else: + taos_V_output = subprocess.getoutput("taos -V | grep version") + + print("taos -V output is: %s" % taos_V_output) + if version in taos_V_output and version in conn.client_info: + version_test_result = True + #print("*******Test Result: Version check passed ************") + +conn.close() +if uninstall: + print("Start to run rmtaos") + leftFile = False + print("Platform: ", system) + + if system == "Linux": + # 创建一个subprocess.Popen对象，并使用stdin和stdout进行交互 + process = subprocess.Popen(['rmtaos'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + # 向子进程发送输入 + process.stdin.write("y\n") + process.stdin.flush() # 确保输入被发送到子进程 + process.stdin.write("I confirm that I would like to delete all data, log and configuration files\n") + process.stdin.flush() # 确保输入被发送到子进程 + # 关闭子进程的stdin，防止它无限期等待更多输入 + process.stdin.close() + # 等待子进程结束 + process.wait() + # 检查目录清除情况 + out = subprocess.getoutput("ls /etc/systemd/system/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/bin/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/bin/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/lib/libtaos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/lib64/libtaos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/include/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/taos") + #print(out) + if "No such file or directory" not in out: + print("Uninstall left some files in /usr/local/taos：%s" % out) + leftFile = True + if not leftFile: + print("*******Test Result: uninstall test passed ************") + + elif system == "Darwin": + # 创建一个subprocess.Popen对象，并使用stdin和stdout进行交互 + process = subprocess.Popen(['sudo', 'rmtaos'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + # 向子进程发送输入 + process.stdin.write("y\n") + process.stdin.flush() # 确保输入被发送到子进程 + process.stdin.write("I confirm that I would like to delete all data, log and configuration files\n") + process.stdin.flush() # 确保输入被发送到子进程 + # 关闭子进程的stdin，防止它无限期等待更多输入 + process.stdin.close() + # 等待子进程结束 + process.wait() + # 检查目录清除情况 + out = subprocess.getoutput("ls /usr/local/bin/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/lib/libtaos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/include/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + #out = subprocess.getoutput("ls /usr/local/Cellar/tdengine/") + #print(out) + #if out: + # print("Uninstall left some files: /usr/local/Cellar/tdengine/%s" % out) + # leftFile = True + #if not leftFile: + # print("*******Test Result: uninstall test passed ************") + + elif system == "Windows": + process = subprocess.Popen(['unins000','/silent'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + process.wait() + time.sleep(10) + out = subprocess.getoutput("ls C:\TDengine") + print(out) + if len(out.split("\n")) > 3: + leftFile = True + print("Uninstall left some files: %s" % out) + +if taosBenchmark_test_result: + print("**********Test Result: taosBenchmark test passed! **********") +if taos_test_result: + print("**********Test Result: taos test passed! **********") +else: + print("!!!!!!!!!!!Test Result: taos test failed! !!!!!!!!!!") +if version_test_result: + print("**********Test Result: version test passed! **********") +else: + print("!!!!!!!!!!!Test Result: version test failed! !!!!!!!!!!") +if not leftFile: + print("**********Test Result: uninstall test passed! **********") +else: + print("!!!!!!!!!!!Test Result: uninstall test failed! !!!!!!!!!!") +if taosBenchmark_test_result and taos_test_result and version_test_result and not leftFile: + sys.exit(0) +else: + sys.exit(1) + diff --git a/packaging/smoke_test/smokeTestJenkinsFile b/packaging/smoke_test/smokeTestJenkinsFile new file mode 100644 index 0000000000..464393d85d --- /dev/null +++ b/packaging/smoke_test/smokeTestJenkinsFile @@ -0,0 +1,380 @@ +def sync_source(branch_name) { + sh ''' + hostname + ip addr|grep 192|awk '{print $2}'|sed "s/\\/.*//" + echo ''' + branch_name + ''' + ''' + sh ''' + cd ${TDENGINE_ROOT_DIR} + set +e + git reset --hard + git fetch || git fetch + git checkout -f '''+branch_name+''' + git reset --hard origin/'''+branch_name+''' + git log | head -n 20 + git clean -fxd + set -e + ''' + return 1 +} +def sync_source_win() { + bat ''' + hostname + taskkill /f /t /im taosd.exe + ipconfig + set + date /t + time /t + ''' + bat ''' + echo %branch_name% + cd %WIN_TDENGINE_ROOT_DIR% + git reset --hard + git fetch || git fetch + git checkout -f ''' + env.BRANCH_NAME + ''' + git reset --hard origin/''' + env.BRANCH_NAME + ''' + git branch + git restore . + git remote prune origin + git pull || git pull + git log | head -n 20 + git clean -fxd + ''' + return 1 +} +pipeline { + agent none + parameters { + choice( + name: 'sourcePath', + choices: ['nas','web'], + description: 'Choice which way to download the installation pacakge;web is Office Web and nas means taos nas server ' + ) + choice( + name: 'verMode', + choices: ['enterprise','community'], + description: 'Choice which types of package you want do check ' + ) + string ( + name:'version', + defaultValue:'3.3.2.0', + description: 'Release version number,eg: 3.0.0.1' + ) + string ( + name:'baseVersion', + defaultValue:'smoking', + description: 'Tnas root path. eg:smoking, 3.3' + ) + choice ( + name:'mode', + choices: ['server','client'], + description: 'Choose which mode of package you want do run ' + ) + choice ( + name:'smoke_branch', + choices: ['test/3.0/smokeTest','test/main/smokeTest','test/3.1/smokeTest'], + description: 'Choose which mode of package you want do run ' + ) + string ( + name:'runPlatforms', + defaultValue:'server_Linux_x64, server_Linux_arm64, server_Windows_x64, server_Mac_x64', + description: 'run package list hotfix usually run: server: server_Linux_x64, server_Linux_arm64 client: client_Linux_x64, client_Linux_arm64 release usually run: enterprise server: server_Linux_x64, server_Linux_arm64, server_Windows_x64 enterprise client: client_Linux_x64, client_Linux_arm64, client_Windows_x64 community server: server_Linux_x64, server_Linux_arm64, server_Mac_x64, server_Mac_arm64(not supported), server_Linux_x64_lite(not supported) community client: client_Linux_x64, client_Linux_arm64, client_Windows_x64, client_Mac_x64, client_Mac_arm64(not supported), client_Linux_x64_lite(not supported)' + ) + } + environment{ + WORK_DIR = "/var/lib/jenkins/workspace" + TDINTERNAL_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal' + TDENGINE_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal/community' + BRANCH_NAME = "${smoke_branch}" + } + stages { + stage ('Start Server for Client Test') { + when { + beforeAgent true + expression { mode == 'client' } + } + agent{label " ubuntu18 "} + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + withEnv(['JENKINS_NODE_COOKIE=dontkillme']) { + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + bash start3NodesServer.sh + ''' + } + } + } + } + stage ('Run SmokeTest') { + parallel { + stage('server_Linux_x64') { + when { + beforeAgent true + allOf { + expression { mode == 'server' } + expression { runPlatforms.contains('server_Linux_x64') } + } + } + agent{label " ubuntu16 "} + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + sh ''' + mkdir -p /var/www/html/${baseVersion}/${version}/${verMode}/json + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 -m pytest test_server.py -v --html=/var/www/html/${baseVersion}/${version}/${verMode}/${mode}_linux_x64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=${verMode} --tVersion=${version} --baseVersion=${baseVersion} --sourcePath=${sourcePath} || true + cp report.json /var/www/html/${baseVersion}/${version}/${verMode}/json/${mode}_linux_x64_report.json + curl "http://192.168.0.176/api/addSmoke?version=${version}&tag=${baseVersion}&type=${verMode}&role=server&build=linux_x64" + ''' + } + } + } + stage('server_Linux_arm64') { + when { + beforeAgent true + allOf { + expression { mode == 'server' } + expression { runPlatforms.contains('server_Linux_arm64') } + } + } + agent{label "worker06_arm64"} + steps { + timeout(time: 60, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f server -l false -c arm64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 -m pytest test_server.py -v --html=${mode}_linux_arm64_report.html --json-report --json-report-file=report.json --timeout=600 --verMode=${verMode} --tVersion=${version} --baseVersion=${baseVersion} --sourcePath=${sourcePath} || true + scp ${mode}_linux_arm64_report.html root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/ + scp report.json root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/json/${mode}_linux_arm64_report.json + curl "http://192.168.0.176/api/addSmoke?version=${version}&tag=${baseVersion}&type=${verMode}&role=server&build=linux_arm64" + ''' + } + } + } + stage ('server_Mac_x64') { + when { + beforeAgent true + allOf { + expression { mode == 'server' } + expression { runPlatforms.contains('server_Mac_x64') } + } + } + agent{label " release_Darwin_x64 "} + environment{ + WORK_DIR = "/Users/zwen/jenkins/workspace" + TDINTERNAL_ROOT_DIR = '/Users/zwen/jenkins/workspace/TDinternal' + TDENGINE_ROOT_DIR = '/Users/zwen/jenkins/workspace/TDinternal/community' + } + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t pkg + python3 -m pytest -v -k linux --html=${mode}_Mac_x64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=${verMode} --tVersion=${version} --baseVersion=${baseVersion} --sourcePath=${sourcePath} || true + scp ${mode}_Mac_x64_report.html root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/ + scp report.json root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/json/${mode}_Mac_x64_report.json + curl "http://192.168.0.176/api/addSmoke?version=${version}&tag=${baseVersion}&type=${verMode}&role=server&build=Mac_x64" + ''' + } + } + } + stage ('server_Mac_arm64') { + when { + beforeAgent true + allOf { + expression { mode == 'server' } + expression { runPlatforms.contains('server_Mac_arm64') } + } + } + agent{label " release_Darwin_arm64 "} + environment{ + WORK_DIR = "/Users/zwen/jenkins/workspace" + TDINTERNAL_ROOT_DIR = '/Users/zwen/jenkins/workspace/TDinternal' + TDENGINE_ROOT_DIR = '/Users/zwen/jenkins/workspace/TDinternal/community' + } + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f server -l false -c arm64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t pkg + python3 -m pytest -v -k linux --html=${mode}_Mac_arm64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=${verMode} --tVersion=${version} --baseVersion=${baseVersion} --sourcePath=${sourcePath} || true + scp ${mode}_Mac_arm64_report.html root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/ + scp report.json root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/json/${mode}_Mac_arm64_report.json + curl "http://192.168.0.176/api/addSmoke?version=${version}&tag=${baseVersion}&type=${verMode}&role=server&build=Mac_arm64" + ''' + } + } + } + stage('server_Windows_x64') { + when { + beforeAgent true + allOf { + expression { mode == 'server' } + expression { runPlatforms.contains('server_Windows_x64') } + } + } + agent{label " windows11 "} + environment{ + WIN_WORK_DIR="C:\\workspace" + WIN_TDINTERNAL_ROOT_DIR="C:\\workspace\\TDinternal" + WIN_TDENGINE_ROOT_DIR="C:\\workspace\\TDinternal\\community" + } + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source_win() + bat ''' + cd %WIN_TDENGINE_ROOT_DIR%\\packaging\\smokeTest + call getAndRunInstaller.bat %baseVersion% %version% %verMode% server + cd %WIN_TDENGINE_ROOT_DIR%\\packaging\\smokeTest + pip3 install -r pytest_require.txt + python3 -m pytest test_server.py -v --html=%mode%_Windows_x64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=%verMode% --tVersion=%version% --baseVersion=%baseVersion% --sourcePath=%sourcePath% + scp %mode%_Windows_x64_report.html root@192.168.0.21:/var/www/html/%baseVersion%/%version%/%verMode%/ + scp report.json root@192.168.0.21:/var/www/html/%baseVersion%/%version%/%verMode%/json/%mode%_Windows_x64_report.json + curl "http://192.168.0.176/api/addSmoke?version=%version%&tag=%baseVersion%&type=%verMode%&role=server&build=Windows_x64" + ''' + } + } + } + stage('client_Linux_x64') { + when { + beforeAgent true + allOf { + expression { mode == 'client' } + expression { runPlatforms.contains('client_Linux_x64') } + } + } + agent{label " ubuntu16 "} + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + sh ''' + mkdir -p /var/www/html/${baseVersion}/${version}/${verMode}/json + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f client -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 -m pytest test_client.py -v --html=/var/www/html/${baseVersion}/${version}/${verMode}/${mode}_linux_x64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=${verMode} --tVersion=${version} --baseVersion=${baseVersion} --sourcePath=${sourcePath} || true + cp report.json /var/www/html/${baseVersion}/${version}/${verMode}/json/${mode}_linux_x64_report.json + curl "http://192.168.0.176/api/addSmoke?version=${version}&tag=${baseVersion}&type=${verMode}&role=client&build=linux_x64" + ''' + } + } + } + stage('client_Linux_arm64') { + when { + beforeAgent true + allOf { + expression { mode == 'client' } + expression { runPlatforms.contains('client_Linux_arm64') } + } + } + agent{label " worker06_arm64 "} + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f client -l false -c arm64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + python3 -m pytest test_client.py -v --html=${mode}_linux_arm64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=${verMode} --tVersion=${version} --baseVersion=${baseVersion} --sourcePath=${sourcePath} || true + scp ${mode}_linux_arm64_report.html root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/ + scp report.json root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/json/${mode}_linux_arm64_report.json + curl "http://192.168.0.176/api/addSmoke?version=${version}&tag=${baseVersion}&type=${verMode}&role=client&build=linux_arm64" + ''' + } + } + } + stage ('client_Mac_x64') { + when { + beforeAgent true + allOf { + expression { mode == 'client' } + expression { runPlatforms.contains('client_Mac_x64') } + } + } + agent{label " release_Darwin_x64 "} + environment{ + WORK_DIR = "/Users/zwen/jenkins/workspace" + TDINTERNAL_ROOT_DIR = '/Users/zwen/jenkins/workspace/TDinternal' + TDENGINE_ROOT_DIR = '/Users/zwen/jenkins/workspace/TDinternal/community' + } + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f client -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t pkg + rm -rf /opt/taos/main/TDinternal/debug/* || true + python3 -m pytest test_client.py -v --html=${mode}_Mac_x64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=${verMode} --tVersion=${version} --baseVersion=${baseVersion} --sourcePath=${sourcePath} || true + scp ${mode}_Mac_x64_report.html root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/ + scp report.json root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/json/${mode}_Mac_x64_report.json + curl "http://192.168.0.176/api/addSmoke?version=${version}&tag=${baseVersion}&type=${verMode}&role=client&build=Mac_x64" + ''' + } + } + } + stage ('client_Mac_arm64') { + when { + beforeAgent true + allOf { + expression { mode == 'client' } + expression { runPlatforms.contains('client_Mac_arm64') } + } + } + agent{label " release_Darwin_arm64 "} + environment{ + WORK_DIR = "/Users/zwen/jenkins/workspace" + TDINTERNAL_ROOT_DIR = '/Users/zwen/jenkins/workspace/TDinternal' + TDENGINE_ROOT_DIR = '/Users/zwen/jenkins/workspace/TDinternal/community' + } + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source("${BRANCH_NAME}") + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging/smokeTest + bash getAndRunInstaller.sh -m ${verMode} -f client -l false -c arm64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t pkg + rm -rf /opt/taos/main/TDinternal/debug/* || true + python3 -m pytest test_client.py -v --html=${mode}_Mac_arm64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=${verMode} --tVersion=${version} --baseVersion=${baseVersion} --sourcePath=${sourcePath} || true + scp ${mode}_Mac_arm64_report.html root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/ + scp report.json root@192.168.0.21:/var/www/html/${baseVersion}/${version}/${verMode}/json/${mode}_Mac_arm64_report.json + curl "http://192.168.0.176/api/addSmoke?version=${version}&tag=${baseVersion}&type=${verMode}&role=client&build=Mac_arm64" + ''' + } + } + } + stage('client_Windows_x64') { + when { + beforeAgent true + allOf { + expression { mode == 'client' } + expression { runPlatforms.contains('client_Windows_x64') } + } + } + agent{label " windows71 "} + environment{ + WIN_WORK_DIR="C:\\workspace" + WIN_TDINTERNAL_ROOT_DIR="C:\\workspace\\TDinternal" + WIN_TDENGINE_ROOT_DIR="C:\\workspace\\TDinternal\\community" + } + steps { + timeout(time: 30, unit: 'MINUTES'){ + sync_source_win() + bat ''' + cd %WIN_TDENGINE_ROOT_DIR%\\packaging\\smokeTest + call getAndRunInstaller.bat %baseVersion% %version% %verMode% client + pip3 install -r pytest_require.txt + python3 -m pytest test_client.py -v --html=%mode%_Windows_x64_report.html --json-report --json-report-file=report.json --timeout=300 --verMode=%verMode% --tVersion=%version% --baseVersion=%baseVersion% --sourcePath=%sourcePath% + scp %mode%_Windows_x64_report.html root@192.168.0.21:/var/www/html/%baseVersion%/%version%/%verMode%/ + scp report.json root@192.168.0.21:/var/www/html/%baseVersion%/%version%/%verMode%/json/%mode%_Windows_x64_report.json + curl "http://192.168.0.176/api/addSmoke?version=%version%&tag=%baseVersion%&type=%verMode%&role=client&build=Windows_x64" + ''' + } + } + } + } + } + } +} \ No newline at end of file diff --git a/packaging/smoke_test/start3NodesServer.sh b/packaging/smoke_test/start3NodesServer.sh new file mode 100644 index 0000000000..b446a467ef --- /dev/null +++ b/packaging/smoke_test/start3NodesServer.sh @@ -0,0 +1,67 @@ +#!/bin/bash +BUILD_ID=dontKillMe + +#******This script setup 3 nodes env for remote client installer test. Only for Linux ********* + +pwd=`pwd` +hostname=`hostname` +if [ -z $JENKINS_HOME ]; then + workdir="${pwd}/cluster" + echo $workdir +else + workdir="${JENKINS_HOME}/workspace/cluster" + echo $workdir +fi + +name="taos" +if command -v prodb ;then + name="prodb" +fi + +# Stop all taosd processes +for(( i=0; i<3; i++)) +do + pid=$(ps -ef | grep ${name}d | grep -v grep | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +done + +# Init 3 dnodes workdir and config file +rm -rf ${workdir} +mkdir ${workdir} +mkdir ${workdir}/output +mkdir ${workdir}/dnode1 +mkdir ${workdir}/dnode1/data +mkdir ${workdir}/dnode1/log +mkdir ${workdir}/dnode1/cfg +touch ${workdir}/dnode1/cfg/${name}.cfg +echo -e "firstEp ${hostname}:6031\nsecondEp ${hostname}:6032\nfqdn ${hostname}\nserverPort 6031\nlogDir ${workdir}/dnode1/log\ndataDir ${workdir}/dnode1/data\n" >> ${workdir}/dnode1/cfg/${name}.cfg + +# Start first node +nohup ${name}d -c ${workdir}/dnode1/cfg/${name}.cfg & > /dev/null +sleep 5 + +${name} -P 6031 -s "CREATE DNODE \`${hostname}:6032\`;CREATE DNODE \`${hostname}:6033\`" + +mkdir ${workdir}/dnode2 +mkdir ${workdir}/dnode2/data +mkdir ${workdir}/dnode2/log +mkdir ${workdir}/dnode2/cfg +touch ${workdir}/dnode2/cfg/${name}.cfg +echo -e "firstEp ${hostname}:6031\nsecondEp ${hostname}:6032\nfqdn ${hostname}\nserverPort 6032\nlogDir ${workdir}/dnode2/log\ndataDir ${workdir}/dnode2/data\n" >> ${workdir}/dnode2/cfg/${name}.cfg + +nohup ${name}d -c ${workdir}/dnode2/cfg/${name}.cfg & > /dev/null +sleep 5 + +mkdir ${workdir}/dnode3 +mkdir ${workdir}/dnode3/data +mkdir ${workdir}/dnode3/log +mkdir ${workdir}/dnode3/cfg +touch ${workdir}/dnode3/cfg/${name}.cfg +echo -e "firstEp ${hostname}:6031\nsecondEp ${hostname}:6032\nfqdn ${hostname}\nserverPort 6033\nlogDir ${workdir}/dnode3/log\ndataDir ${workdir}/dnode3/data\n" >> ${workdir}/dnode3/cfg/${name}.cfg + +nohup ${name}d -c ${workdir}/dnode3/cfg/${name}.cfg & > /dev/null +sleep 5 + +${name} -P 6031 -s "CREATE MNODE ON DNODE 2;CREATE MNODE ON DNODE 3;" \ No newline at end of file diff --git a/packaging/smoke_test/test_client.py b/packaging/smoke_test/test_client.py new file mode 100644 index 0000000000..0b1003e370 --- /dev/null +++ b/packaging/smoke_test/test_client.py @@ -0,0 +1,137 @@ +import pytest +import subprocess +import os +import sys +import platform +import getopt +import re +import time +import taos +from versionCheckAndUninstallforPytest import UninstallTaos + +# python3 smokeTestClient.py -h 192.168.0.22 -P 6031 -v ${version} -u + +OEM = ["ProDB"] + + +@pytest.fixture(scope="module") +def get_config(request): + verMode = request.config.getoption("--verMode") + taosVersion = request.config.getoption("--tVersion") + baseVersion = request.config.getoption("--baseVersion") + sourcePath = request.config.getoption("--sourcePath") + config = { + "verMode": verMode, + "taosVersion": taosVersion, + "baseVersion": baseVersion, + "sourcePath": sourcePath, + "system": platform.system(), + "arch": platform.machine(), + "serverHost": "192.168.0.22", + "serverPort": 6031, + "databaseName": re.sub(r'[^a-zA-Z0-9]', '', subprocess.getoutput("hostname")).lower() + } + return config + + +@pytest.fixture(scope="module") +def setup_module(get_config): + config = get_config + # install taospy + if config["system"] == 'Windows': + taospy_version = subprocess.getoutput("pip3 show taospy|findstr Version") + else: + taospy_version = subprocess.getoutput("pip3 show taospy|grep Version| awk -F ':' '{print $2}' ") + + print("taospy version %s " % taospy_version) + if taospy_version == "": + subprocess.getoutput("pip3 install git+https://github.com/taosdata/taos-connector-python.git") + print("install taos python connector") + else: + subprocess.getoutput("pip3 install taospy") + + +def get_connect(host, port, database=None): + conn = taos.connect(host=host, + user="root", + password="taosdata", + database=database, + port=port, + timezone="Asia/Shanghai") # default your host's timezone + return conn + + +def run_cmd(command): + print("CMD: %s" % command) + result = subprocess.run(command, capture_output=True, text=True, shell=True) + print("STDOUT:", result.stdout) + print("STDERR:", result.stderr) + print("Return Code:", result.returncode) + assert result.returncode == 0 + return result + + +class TestClient: + @pytest.mark.all + def test_basic(self, get_config, setup_module): + config = get_config + name = "taos" + + if config["baseVersion"] in OEM: + name = config["baseVersion"].lower() + if config["baseVersion"] in OEM and config["system"] == 'Windows': + cmd = f'{name} -s "create database {config["databaseName"]};" -h {config["serverHost"]} -P {config["serverPort"]}' + run_cmd(cmd) + cmd = f'{name} -s "CREATE STABLE {config["databaseName"]}.meters (`ts` TIMESTAMP,`current` FLOAT, `phase` FLOAT) TAGS (`groupid` INT, `location` VARCHAR(24));" -h {config["serverHost"]} -P {config["serverPort"]}' + run_cmd(cmd) + else: + cmd = f'{name}Benchmark -y -a 3 -n 100 -t 100 -d {config["databaseName"]} -h {config["serverHost"]} -P {config["serverPort"]} &' + run_cmd(cmd) + # os.system("taosBenchmark -y -a 3 -n 100 -t 100 -d %s -h %s -P %d" % (databaseName, serverHost, serverPort)) + time.sleep(5) + conn = get_connect(config["serverHost"], config["serverPort"], config["databaseName"]) + sql = "SELECT count(*) from meters" + result: taos.TaosResult = conn.query(sql) + data = result.fetch_all() + print("SQL: %s" % sql) + print("Result: %s" % data) + if config["system"] == 'Windows' and config["baseVersion"] in OEM: + pass + elif data[0][0] != 10000: + raise f"{name}Benchmark work not as expected " + # drop database of test + cmd = f'{name} -s "drop database {config["databaseName"]};" -h {config["serverHost"]} -P {config["serverPort"]}' + result = run_cmd(cmd) + assert "Drop OK" in result.stdout + conn.close() + + @pytest.mark.all + def test_version(self, get_config, setup_module): + config = get_config + conn = get_connect(config["serverHost"], config["serverPort"]) + server_version = conn.server_info + print("server_version: ", server_version) + client_version = conn.client_info + print("client_version: ", client_version) + name = "taos" + if config["baseVersion"] in OEM: + name = config["baseVersion"].lower() + if config["system"] == "Windows": + taos_V_output = subprocess.getoutput(f"{name} -V | findstr version") + else: + taos_V_output = subprocess.getoutput(f"{name} -V | grep version") + assert config["taosVersion"] in taos_V_output + assert config["taosVersion"] in client_version + if config["taosVersion"] not in server_version: + print("warning: client version is not same as server version") + conn.close() + + @pytest.mark.all + def test_uninstall(self, get_config, setup_module): + config = get_config + name = "taos" + if config["baseVersion"] in OEM: + name = config["baseVersion"].lower() + subprocess.getoutput("rm /usr/local/bin/taos") + subprocess.getoutput("pkill taosd") + UninstallTaos(config["taosVersion"], config["verMode"], True, name) diff --git a/packaging/smoke_test/test_server.py b/packaging/smoke_test/test_server.py new file mode 100644 index 0000000000..768cd2a962 --- /dev/null +++ b/packaging/smoke_test/test_server.py @@ -0,0 +1,238 @@ +import pytest +import subprocess +import os +from versionCheckAndUninstallforPytest import UninstallTaos +import platform +import re +import time +import signal + +system = platform.system() +current_path = os.path.abspath(os.path.dirname(__file__)) +if system == 'Windows': + with open(r"%s\test_server_windows_case" % current_path) as f: + cases = f.read().splitlines() +else: + with open("%s/test_server_unix_case" % current_path) as f: + cases = f.read().splitlines() + +OEM = ["ProDB"] + + +@pytest.fixture(scope="module") +def get_config(request): + verMode = request.config.getoption("--verMode") + taosVersion = request.config.getoption("--tVersion") + baseVersion = request.config.getoption("--baseVersion") + sourcePath = request.config.getoption("--sourcePath") + config = { + "verMode": verMode, + "taosVersion": taosVersion, + "baseVersion": baseVersion, + "sourcePath": sourcePath, + "system": platform.system(), + "arch": platform.machine() + } + return config + + +@pytest.fixture(scope="module") +def setup_module(get_config): + def run_cmd(command): + print("CMD:", command) + result = subprocess.run(command, capture_output=True, text=False, shell=False) + print("STDOUT:", result.stdout) + print("STDERR:", result.stderr) + print("Return Code:", result.returncode) + assert result.returncode == 0 + return result + + # setup before module tests + config = get_config + # bash getAndRunInstaller.sh -m ${verMode} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar + # t = "tar" + # if config["system"] == "Darwin": + # t = "pkg" + # cmd = "bash getAndRunInstaller.sh -m %s -f server -l false -c x64 -v %s -o %s -s %s -t %s" % ( + # config["verMode"], config["taosVersion"], config["baseVersion"], config["sourcePath"], t) + # run_cmd(cmd) + if config["system"] == "Windows": + cmd = r"mkdir ..\..\debug\build\bin" + else: + cmd = "mkdir -p ../../debug/build/bin/" + subprocess.getoutput(cmd) + if config["system"] == "Linux": # add tmq_sim + cmd = "cp -rf ../../../debug/build/bin/tmq_sim ../../debug/build/bin/." + subprocess.getoutput(cmd) + if config["system"] == "Darwin": + cmd = "sudo cp -rf /usr/local/bin/taos* ../../debug/build/bin/" + elif config["system"] == "Windows": + cmd = r"xcopy C:\TDengine\taos*.exe ..\..\debug\build\bin /Y" + else: + if config["baseVersion"] in OEM: + cmd = '''sudo find /usr/bin -name 'prodb*' -exec sh -c 'for file; do cp "$file" "../../debug/build/bin/taos${file##/usr/bin/%s}"; done' sh {} +''' % ( + config["baseVersion"].lower()) + else: + cmd = "sudo cp /usr/bin/taos* ../../debug/build/bin/" + run_cmd(cmd) + if config["baseVersion"] in OEM: # mock OEM + cmd = "sed -i 's/taos.cfg/%s.cfg/g' ../../tests/pytest/util/dnodes.py" % config["baseVersion"].lower() + run_cmd(cmd) + cmd = "sed -i 's/taosdlog.0/%sdlog.0/g' ../../tests/pytest/util/dnodes.py" % config["baseVersion"].lower() + run_cmd(cmd) + cmd = "sed -i 's/taos.cfg/%s.cfg/g' ../../tests/army/frame/server/dnode.py" % config["baseVersion"].lower() + run_cmd(cmd) + cmd = "sed -i 's/taosdlog.0/%sdlog.0/g' ../../tests/army/frame/server/dnode.py" % config["baseVersion"].lower() + run_cmd(cmd) + cmd = "ln -s /usr/bin/prodb /usr/local/bin/taos" + subprocess.getoutput(cmd) + + # yield + # + # name = "taos" + # if config["baseVersion"] in OEM: + # name = config["baseVersion"].lower() + # subprocess.getoutput("rm /usr/local/bin/taos") + # subprocess.getoutput("pkill taosd") + # UninstallTaos(config["taosVersion"], config["verMode"], True, name) + + +# use pytest fixture to exec case +@pytest.fixture(params=cases) +def run_command(request): + commands = request.param + if commands.strip().startswith("#"): + pytest.skip("This case has been marked as skipped") + d, command = commands.strip().split(",") + if system == "Windows": + cmd = r"cd %s\..\..\tests\%s && %s" % (current_path, d, command) + else: + cmd = "cd %s/../../tests/%s&&sudo %s" % (current_path, d, command) + print(cmd) + result = subprocess.run(cmd, capture_output=True, text=True, shell=True) + return { + "command": command, + "stdout": result.stdout, + "stderr": result.stderr, + "returncode": result.returncode + } + + +class TestServer: + @pytest.mark.all + def test_taosd_up(self, setup_module): + # start process + if system == 'Windows': + subprocess.getoutput("taskkill /IM taosd.exe /F") + cmd = "..\\..\\debug\\build\\bin\\taosd.exe" + else: + subprocess.getoutput("pkill taosd") + cmd = "../../debug/build/bin/taosd" + process = subprocess.Popen( + [cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + # monitor output + while True: + line = process.stdout.readline() + if line: + print(line.strip()) + if "succeed to write dnode" in line: + time.sleep(15) + # 发送终止信号 + os.kill(process.pid, signal.SIGTERM) + break + + @pytest.mark.all + def test_execute_cases(self, setup_module, run_command): + # assert the result + if run_command['returncode'] != 0: + print(f"Running command: {run_command['command']}") + print("STDOUT:", run_command['stdout']) + print("STDERR:", run_command['stderr']) + print("Return Code:", run_command['returncode']) + else: + print(f"Running command: {run_command['command']}") + if len(run_command['stdout']) > 1000: + print("STDOUT:", run_command['stdout'][:1000] + "...") + else: + print("STDOUT:", run_command['stdout']) + print("STDERR:", run_command['stderr']) + print("Return Code:", run_command['returncode']) + + assert run_command[ + 'returncode'] == 0, f"Command '{run_command['command']}' failed with return code {run_command['returncode']}" + + @pytest.mark.all + @pytest.mark.check_version + def test_check_version(self, get_config, setup_module): + config = get_config + databaseName = re.sub(r'[^a-zA-Z0-9]', '', subprocess.getoutput("hostname")).lower() + # install taospy + taospy_version = "" + system = config["system"] + version = config["taosVersion"] + verMode = config["verMode"] + if system == 'Windows': + taospy_version = subprocess.getoutput("pip3 show taospy|findstr Version") + else: + taospy_version = subprocess.getoutput("pip3 show taospy|grep Version| awk -F ':' '{print $2}' ") + + print("taospy version %s " % taospy_version) + if taospy_version == "": + subprocess.getoutput("pip3 install git+https://github.com/taosdata/taos-connector-python.git") + print("install taos python connector") + else: + subprocess.getoutput("pip3 install taospy") + + # start taosd server + if system == 'Windows': + cmd = ["C:\\TDengine\\start-all.bat"] + # elif system == 'Linux': + # cmd = "systemctl start taosd".split(' ') + else: + # cmd = "sudo launchctl start com.tdengine.taosd".split(' ') + cmd = "start-all.sh" + process_out = subprocess.Popen(cmd, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + print(cmd) + time.sleep(5) + + import taos + conn = taos.connect() + check_list = {} + check_list["server_version"] = conn.server_info + check_list["client_version"] = conn.client_info + # Execute sql get version info + result: taos.TaosResult = conn.query("SELECT server_version()") + check_list["select_server"] = result.fetch_all()[0][0] + result: taos.TaosResult = conn.query("SELECT client_version()") + check_list["select_client"] = result.fetch_all()[0][0] + conn.close() + + binary_files = ["taos", "taosd", "taosadapter", "taoskeeper", "taosBenchmark"] + if verMode.lower() == "enterprise": + binary_files.append("taosx") + if config["baseVersion"] in OEM: + binary_files = [i.replace("taos", config["baseVersion"].lower()) for i in binary_files] + if system == "Windows": + for i in binary_files: + check_list[i] = subprocess.getoutput("%s -V | findstr version" % i) + else: + for i in binary_files: + check_list[i] = subprocess.getoutput("%s -V | grep version | awk -F ' ' '{print $3}'" % i) + for i in check_list: + print("%s version is: %s" % (i, check_list[i])) + assert version in check_list[i] + + @pytest.mark.all + def test_uninstall(self, get_config, setup_module): + config = get_config + name = "taos" + if config["baseVersion"] in OEM: + name = config["baseVersion"].lower() + subprocess.getoutput("rm /usr/local/bin/taos") + subprocess.getoutput("pkill taosd") + UninstallTaos(config["taosVersion"], config["verMode"], True, name) diff --git a/packaging/smoke_test/test_server_unix_case b/packaging/smoke_test/test_server_unix_case new file mode 100644 index 0000000000..1bbde10932 --- /dev/null +++ b/packaging/smoke_test/test_server_unix_case @@ -0,0 +1,10 @@ +system-test,python3 ./test.py -f 2-query/join.py +system-test,python3 ./test.py -f 1-insert/insert_column_value.py +system-test,python3 ./test.py -f 2-query/primary_ts_base_5.py +system-test,python3 ./test.py -f 2-query/case_when.py +system-test,python3 ./test.py -f 2-query/partition_limit_interval.py +system-test,python3 ./test.py -f 2-query/fill.py +army,python3 ./test.py -f query/query_basic.py -N 3 +system-test,python3 ./test.py -f 7-tmq/basic5.py +system-test,python3 ./test.py -f 8-stream/stream_basic.py +system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 \ No newline at end of file diff --git a/packaging/smoke_test/test_server_windows_case b/packaging/smoke_test/test_server_windows_case new file mode 100644 index 0000000000..e64213b1ee --- /dev/null +++ b/packaging/smoke_test/test_server_windows_case @@ -0,0 +1,2 @@ +system-test,python3 .\test.py -f 0-others\taosShell.py +system-test,python3 .\test.py -f 6-cluster\5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py -N 6 -M 3 \ No newline at end of file diff --git a/packaging/smoke_test/versionCheckAndUninstall.py b/packaging/smoke_test/versionCheckAndUninstall.py new file mode 100644 index 0000000000..80dea9a15f --- /dev/null +++ b/packaging/smoke_test/versionCheckAndUninstall.py @@ -0,0 +1,260 @@ +#!/usr/bin/python +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### +# install pip +# pip install src/connector/python/ + +# -*- coding: utf-8 -*- +import sys, os +import re +import platform +import getopt +import subprocess +# from this import d +import time + +# input for server + +opts, args = getopt.gnu_getopt(sys.argv[1:], 'v:m:u', ['version=', 'verMode=']) +serverHost = "" +serverPort = 0 +version = "" +uninstall = False +verMode = "" +for key, value in opts: + if key in ['--help']: + print('A collection of test cases written using Python') + print('-v test client version') + print('-u test uninstall process, will uninstall TDengine') + sys.exit(0) + + if key in ['-v']: + version = value + if key in ['-u']: + uninstall = True + if key in ['-m']: + verMode = value +if not version: + print("No version specified, will not run version check.") + + +system = platform.system() +arch = platform.machine() + +databaseName = re.sub(r'[^a-zA-Z0-9]', '', subprocess.getoutput("hostname")).lower() +# install taospy +taospy_version = "" +if system == 'Windows': + taospy_version = subprocess.getoutput("pip3 show taospy|findstr Version") +else: + taospy_version = subprocess.getoutput("pip3 show taospy|grep Version| awk -F ':' '{print $2}' ") + +print("taospy version %s " % taospy_version) +if taospy_version == "": + subprocess.getoutput("pip3 install git+https://github.com/taosdata/taos-connector-python.git") + print("install taos python connector") +else: + subprocess.getoutput("pip3 install taospy") + +# start taosd server +if system == 'Windows': + cmd = ["C:\\TDengine\\start-all.bat"] +elif system == 'Linux': + cmd = "systemctl start taosd".split(' ') +else: + cmd = "sudo launchctl start com.tdengine.taosd".split(' ') +process_out = subprocess.Popen(cmd, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) +print(cmd) +time.sleep(5) + +#get taosc version info +version_test_result = False +if version: + import taos + conn = taos.connect() + server_version = conn.server_info + print("server_version", server_version) + client_version = conn.client_info + print("client_version", client_version) + # Execute sql get version info + result: taos.TaosResult = conn.query("SELECT server_version()") + select_server = result.fetch_all()[0][0] + print("SELECT server_version():" + select_server) + result: taos.TaosResult = conn.query("SELECT client_version()") + select_client = result.fetch_all()[0][0] + print("SELECT client_version():" + select_client) + conn.close() + + taos_V_output = "" + taosd_V_output = "" + taosadapter_V_output = "" + taoskeeper_V_output = "" + taosx_V_output = "" + taosB_V_output = "" + taosxVersion = False + if system == "Windows": + taos_V_output = subprocess.getoutput("taos -V | findstr version") + taosd_V_output = subprocess.getoutput("taosd -V | findstr version") + taosadapter_V_output = subprocess.getoutput("taosadapter -V | findstr version") + taoskeeper_V_output = subprocess.getoutput("taoskeeper -V | findstr version") + taosB_V_output = subprocess.getoutput("taosBenchmark -V | findstr version") + if verMode == "Enterprise": + taosx_V_output = subprocess.getoutput("taosx -V | findstr version") + else: + taos_V_output = subprocess.getoutput("taos -V | grep version | awk -F ' ' '{print $3}'") + taosd_V_output = subprocess.getoutput("taosd -V | grep version | awk -F ' ' '{print $3}'") + taosadapter_V_output = subprocess.getoutput("taosadapter -V | grep version | awk -F ' ' '{print $3}'") + taoskeeper_V_output = subprocess.getoutput("taoskeeper -V | grep version | awk -F ' ' '{print $3}'") + taosB_V_output = subprocess.getoutput("taosBenchmark -V | grep version | awk -F ' ' '{print $3}'") + if verMode == "Enterprise": + taosx_V_output = subprocess.getoutput("taosx -V | grep version | awk -F ' ' '{print $3}'") + + print("taos -V output is: %s" % taos_V_output) + print("taosd -V output is: %s" % taosd_V_output) + print("taosadapter -V output is: %s" % taosadapter_V_output) + print("taoskeeper -V output is: %s" % taoskeeper_V_output) + print("taosBenchmark -V output is: %s" % taosB_V_output) + if verMode == "Enterprise": + print("taosx -V output is: %s" % taosx_V_output) + taosxVersion = version in taosx_V_output + else: + taosxVersion = True + if (version in client_version + and version in server_version + and version in select_server + and version in select_client + and version in taos_V_output + and version in taosd_V_output + and version in taosadapter_V_output + and version in taoskeeper_V_output + and version in taosB_V_output + and taosxVersion + ): + version_test_result = True +leftFile = False +if uninstall: + print("Start to run rmtaos") + print("Platform: ", system) + # stop taosd server + if system == 'Windows': + cmd = "C:\\TDengine\\stop_all.bat" + elif system == 'Linux': + cmd = "systemctl stop taosd" + else: + cmd = "sudo launchctl stop com.tdengine.taosd" + process_out = subprocess.getoutput(cmd) + print(cmd) + time.sleep(10) + if system == "Linux": + # 创建一个subprocess.Popen对象，并使用stdin和stdout进行交互 + process = subprocess.Popen(['rmtaos'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + # 向子进程发送输入 + process.stdin.write("y\n") + process.stdin.flush() # 确保输入被发送到子进程 + process.stdin.write("I confirm that I would like to delete all data, log and configuration files\n") + process.stdin.flush() # 确保输入被发送到子进程 + # 关闭子进程的stdin，防止它无限期等待更多输入 + process.stdin.close() + # 等待子进程结束 + process.wait() + # 检查目录清除情况 + out = subprocess.getoutput("ls /etc/systemd/system/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/bin/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/bin/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/lib/libtaos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/lib64/libtaos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/include/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/taos") + #print(out) + if "No such file or directory" not in out: + print("Uninstall left some files in /usr/local/taos：%s" % out) + leftFile = True + if not leftFile: + print("*******Test Result: uninstall test passed ************") + + elif system == "Darwin": + # 创建一个subprocess.Popen对象，并使用stdin和stdout进行交互 + process = subprocess.Popen(['sudo', 'rmtaos'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + # 向子进程发送输入 + process.stdin.write("y\n") + process.stdin.flush() # 确保输入被发送到子进程 + process.stdin.write("I confirm that I would like to delete all data, log and configuration files\n") + process.stdin.flush() # 确保输入被发送到子进程 + # 关闭子进程的stdin，防止它无限期等待更多输入 + process.stdin.close() + # 等待子进程结束 + process.wait() + # 检查目录清除情况 + out = subprocess.getoutput("ls /usr/local/bin/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/lib/libtaos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/include/taos*") + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + #out = subprocess.getoutput("ls /usr/local/Cellar/tdengine/") + #print(out) + #if out: + # print("Uninstall left some files: /usr/local/Cellar/tdengine/%s" % out) + # leftFile = True + #if not leftFile: + # print("*******Test Result: uninstall test passed ************") + + elif system == "Windows": + process = subprocess.Popen(['unins000','/silent'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + process.wait() + time.sleep(10) + out = subprocess.getoutput("ls C:\TDengine") + print(out) + if len(out.split("\n")) > 3: + leftFile = True + print("Uninstall left some files: %s" % out) + +if version_test_result: + print("**********Test Result: version test passed! **********") +else: + print("!!!!!!!!!!!Test Result: version test failed! !!!!!!!!!!") +if not leftFile: + print("**********Test Result: uninstall test passed! **********") +else: + print("!!!!!!!!!!!Test Result: uninstall test failed! !!!!!!!!!!") +if version_test_result and not leftFile: + sys.exit(0) +else: + sys.exit(1) + diff --git a/packaging/smoke_test/versionCheckAndUninstallforPytest.py b/packaging/smoke_test/versionCheckAndUninstallforPytest.py new file mode 100644 index 0000000000..5b75219554 --- /dev/null +++ b/packaging/smoke_test/versionCheckAndUninstallforPytest.py @@ -0,0 +1,137 @@ +#!/usr/bin/python +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### +# install pip +# pip install src/connector/python/ + +# -*- coding: utf-8 -*- +import sys, os +import re +import platform +import getopt +import subprocess +# from this import d +import time +from lib import run_cmd + + +# input for server +def UninstallTaos(version, verMode, uninstall, name): + if not version: + raise "No version specified, will not run version check." + + system = platform.system() + arch = platform.machine() + leftFile = False + if uninstall: + print("Start to run rm%s" % name) + print("Platform: ", system) + # stop taosd server + if system == 'Windows': + cmd = "C:\\TDengine\\stop_all.bat" + else: + cmd = "stop_all.sh" + process_out = subprocess.getoutput(cmd) + print(cmd) + time.sleep(5) + print("start to rm%s" % name) + if system == "Linux": + # 启动命令 + process = subprocess.Popen(['rm%s' % name], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) + + # 发送交互输入 + stdout, stderr = process.communicate( + input="y\nI confirm that I would like to delete all data, log and configuration files\n") + + # 打印输出（可选） + print(stdout) + print(stderr) + # 检查目录清除情况 + out = subprocess.getoutput("ls /etc/systemd/system/%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/bin/%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/bin/%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/lib/lib%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/lib64/lib%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/include/%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/%s" % name) + # print(out) + if "No such file or directory" not in out: + print("Uninstall left some files in /usr/local/%s：%s" % (name, out)) + leftFile = True + if not leftFile: + print("*******Test Result: uninstall test passed ************") + + elif system == "Darwin": + # 创建一个subprocess.Popen对象，并使用stdin和stdout进行交互 + process = subprocess.Popen(['sudo', 'rm%s' % name], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + # 向子进程发送输入 + process.stdin.write("y\n") + process.stdin.flush() # 确保输入被发送到子进程 + process.stdin.write("I confirm that I would like to delete all data, log and configuration files\n") + process.stdin.flush() # 确保输入被发送到子进程 + # 关闭子进程的stdin，防止它无限期等待更多输入 + process.stdin.close() + # 等待子进程结束 + process.wait() + # 检查目录清除情况 + out = subprocess.getoutput("ls /usr/local/bin/%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/lib/lib%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + out = subprocess.getoutput("ls /usr/local/include/%s*" % name) + if "No such file or directory" not in out: + print("Uninstall left some files: %s" % out) + leftFile = True + # out = subprocess.getoutput("ls /usr/local/Cellar/tdengine/") + # print(out) + # if out: + # print("Uninstall left some files: /usr/local/Cellar/tdengine/%s" % out) + # leftFile = True + # if not leftFile: + # print("*******Test Result: uninstall test passed ************") + + elif system == "Windows": + process = subprocess.Popen(['unins000', '/silent'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + process.wait() + time.sleep(10) + for file in ["C:\TDengine\\taos.exe", "C:\TDengine\\unins000.exe", "C:\ProDB\prodb.exe", + "C:\ProDB\\unins000.exe"]: + if os.path.exists(file): + leftFile = True + if leftFile: + raise "uninstall %s fail, please check" % name + else: + print("**********Test Result: uninstall test passed! **********") diff --git a/tests/system-test/test.py b/tests/system-test/test.py index fb3357a2b9..0d40544be8 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -689,6 +689,9 @@ if __name__ == "__main__": if conn is not None: conn.close() if asan: - #tdDnodes.StopAllSigint() + # tdDnodes.StopAllSigint() tdLog.info("Address sanitizer mode finished") + else: + tdDnodes.stopAll() + tdLog.info("stop all td process finished") sys.exit(0) From 81d321a62f7417048f5f90fc1f6fc7ade726de89 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 7 Nov 2024 11:36:59 +0800 Subject: [PATCH 07/44] test:modify somketest dir name --- packaging/{smoke_test => smokeTest}/conftest.py | 0 .../debRpmAutoInstall.sh | 0 .../getAndRunInstaller.bat | 0 .../getAndRunInstaller.sh | 0 packaging/{smoke_test => smokeTest}/lib.py | 0 packaging/{smoke_test => smokeTest}/main.py | 0 packaging/{smoke_test => smokeTest}/runCases.bat | 0 packaging/{smoke_test => smokeTest}/runCases.sh | 0 .../{smoke_test => smokeTest}/smokeTestClient.py | 0 .../smokeTestJenkinsFile | 0 .../start3NodesServer.sh | 0 .../{smoke_test => smokeTest}/test_client.py | 0 .../{smoke_test => smokeTest}/test_server.py | 0 .../test_server_unix_case | 0 .../test_server_windows_case | 0 .../versionCheckAndUninstall.py | 0 .../versionCheckAndUninstallforPytest.py | 0 packaging/smoke_test/pytest_require.txt | 16 ---------------- 18 files changed, 16 deletions(-) rename packaging/{smoke_test => smokeTest}/conftest.py (100%) rename packaging/{smoke_test => smokeTest}/debRpmAutoInstall.sh (100%) rename packaging/{smoke_test => smokeTest}/getAndRunInstaller.bat (100%) rename packaging/{smoke_test => smokeTest}/getAndRunInstaller.sh (100%) rename packaging/{smoke_test => smokeTest}/lib.py (100%) rename packaging/{smoke_test => smokeTest}/main.py (100%) rename packaging/{smoke_test => smokeTest}/runCases.bat (100%) rename packaging/{smoke_test => smokeTest}/runCases.sh (100%) rename packaging/{smoke_test => smokeTest}/smokeTestClient.py (100%) rename packaging/{smoke_test => smokeTest}/smokeTestJenkinsFile (100%) rename packaging/{smoke_test => smokeTest}/start3NodesServer.sh (100%) rename packaging/{smoke_test => smokeTest}/test_client.py (100%) rename packaging/{smoke_test => smokeTest}/test_server.py (100%) rename packaging/{smoke_test => smokeTest}/test_server_unix_case (100%) rename packaging/{smoke_test => smokeTest}/test_server_windows_case (100%) rename packaging/{smoke_test => smokeTest}/versionCheckAndUninstall.py (100%) rename packaging/{smoke_test => smokeTest}/versionCheckAndUninstallforPytest.py (100%) delete mode 100644 packaging/smoke_test/pytest_require.txt diff --git a/packaging/smoke_test/conftest.py b/packaging/smokeTest/conftest.py similarity index 100% rename from packaging/smoke_test/conftest.py rename to packaging/smokeTest/conftest.py diff --git a/packaging/smoke_test/debRpmAutoInstall.sh b/packaging/smokeTest/debRpmAutoInstall.sh similarity index 100% rename from packaging/smoke_test/debRpmAutoInstall.sh rename to packaging/smokeTest/debRpmAutoInstall.sh diff --git a/packaging/smoke_test/getAndRunInstaller.bat b/packaging/smokeTest/getAndRunInstaller.bat similarity index 100% rename from packaging/smoke_test/getAndRunInstaller.bat rename to packaging/smokeTest/getAndRunInstaller.bat diff --git a/packaging/smoke_test/getAndRunInstaller.sh b/packaging/smokeTest/getAndRunInstaller.sh similarity index 100% rename from packaging/smoke_test/getAndRunInstaller.sh rename to packaging/smokeTest/getAndRunInstaller.sh diff --git a/packaging/smoke_test/lib.py b/packaging/smokeTest/lib.py similarity index 100% rename from packaging/smoke_test/lib.py rename to packaging/smokeTest/lib.py diff --git a/packaging/smoke_test/main.py b/packaging/smokeTest/main.py similarity index 100% rename from packaging/smoke_test/main.py rename to packaging/smokeTest/main.py diff --git a/packaging/smoke_test/runCases.bat b/packaging/smokeTest/runCases.bat similarity index 100% rename from packaging/smoke_test/runCases.bat rename to packaging/smokeTest/runCases.bat diff --git a/packaging/smoke_test/runCases.sh b/packaging/smokeTest/runCases.sh similarity index 100% rename from packaging/smoke_test/runCases.sh rename to packaging/smokeTest/runCases.sh diff --git a/packaging/smoke_test/smokeTestClient.py b/packaging/smokeTest/smokeTestClient.py similarity index 100% rename from packaging/smoke_test/smokeTestClient.py rename to packaging/smokeTest/smokeTestClient.py diff --git a/packaging/smoke_test/smokeTestJenkinsFile b/packaging/smokeTest/smokeTestJenkinsFile similarity index 100% rename from packaging/smoke_test/smokeTestJenkinsFile rename to packaging/smokeTest/smokeTestJenkinsFile diff --git a/packaging/smoke_test/start3NodesServer.sh b/packaging/smokeTest/start3NodesServer.sh similarity index 100% rename from packaging/smoke_test/start3NodesServer.sh rename to packaging/smokeTest/start3NodesServer.sh diff --git a/packaging/smoke_test/test_client.py b/packaging/smokeTest/test_client.py similarity index 100% rename from packaging/smoke_test/test_client.py rename to packaging/smokeTest/test_client.py diff --git a/packaging/smoke_test/test_server.py b/packaging/smokeTest/test_server.py similarity index 100% rename from packaging/smoke_test/test_server.py rename to packaging/smokeTest/test_server.py diff --git a/packaging/smoke_test/test_server_unix_case b/packaging/smokeTest/test_server_unix_case similarity index 100% rename from packaging/smoke_test/test_server_unix_case rename to packaging/smokeTest/test_server_unix_case diff --git a/packaging/smoke_test/test_server_windows_case b/packaging/smokeTest/test_server_windows_case similarity index 100% rename from packaging/smoke_test/test_server_windows_case rename to packaging/smokeTest/test_server_windows_case diff --git a/packaging/smoke_test/versionCheckAndUninstall.py b/packaging/smokeTest/versionCheckAndUninstall.py similarity index 100% rename from packaging/smoke_test/versionCheckAndUninstall.py rename to packaging/smokeTest/versionCheckAndUninstall.py diff --git a/packaging/smoke_test/versionCheckAndUninstallforPytest.py b/packaging/smokeTest/versionCheckAndUninstallforPytest.py similarity index 100% rename from packaging/smoke_test/versionCheckAndUninstallforPytest.py rename to packaging/smokeTest/versionCheckAndUninstallforPytest.py diff --git a/packaging/smoke_test/pytest_require.txt b/packaging/smoke_test/pytest_require.txt deleted file mode 100644 index eadbe92470..0000000000 --- a/packaging/smoke_test/pytest_require.txt +++ /dev/null @@ -1,16 +0,0 @@ -pytest-html -pytest-json-report -pytest-timeout -taospy -numpy -fabric2 -psutil -pandas -toml -distro -requests -pexpect -faker -pyopenssl -taos-ws-py -taospy \ No newline at end of file From 04565b23efdf7c463bba6c005c0b85e62fc800c6 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 7 Nov 2024 13:36:53 +0800 Subject: [PATCH 08/44] Parameter validation --- include/common/ttime.h | 3 ++- include/os/osTime.h | 2 +- source/common/src/ttime.c | 2 +- source/libs/function/src/builtins.c | 8 ++++++-- source/libs/parser/src/parInsertSql.c | 2 +- source/os/src/osTime.c | 13 +++++++++--- source/util/src/tlog.c | 29 ++++++++++++++++++++++----- 7 files changed, 45 insertions(+), 14 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index 65bb763b1f..1ffcc29eca 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -62,7 +62,8 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { int64_t factor = (precision == TSDB_TIME_PRECISION_MILLI) ? 1000 : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000 : 1000000000; - time_t t = taosTime(NULL); + time_t t; + (void) taosTime(&t); struct tm tm; (void) taosLocalTime(&t, &tm, NULL, 0); tm.tm_hour = 0; diff --git a/include/os/osTime.h b/include/os/osTime.h index 5d74146e9c..7a65efe28d 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -93,7 +93,7 @@ static FORCE_INLINE int64_t taosGetMonoTimestampMs() { char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm); struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf, int32_t bufSize); struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst); -time_t taosTime(time_t *t); +int32_t taosTime(time_t *t); time_t taosMktime(struct tm *timep); int64_t user_mktime64(const uint32_t year, const uint32_t mon, const uint32_t day, const uint32_t hour, const uint32_t min, const uint32_t sec, int64_t time_zone); diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 75624593d9..ecdb3de9a2 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -30,7 +30,7 @@ static int64_t m_deltaUtc = 0; void deltaToUtcInitOnce() { struct tm tm = {0}; - if (taosStrpTime("1970-01-01 00:00:00", (const char*)("%Y-%m-%d %H:%M:%S"), &tm) != 0) { + if (taosStrpTime("1970-01-01 00:00:00", (const char*)("%Y-%m-%d %H:%M:%S"), &tm) == NULL) { uError("failed to parse time string"); } m_deltaUtc = (int64_t)taosMktime(&tm); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 552933dcad..b369ee794c 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -188,7 +188,11 @@ static int32_t countTrailingSpaces(const SValueNode* pVal, bool isLtrim) { static int32_t addTimezoneParam(SNodeList* pList) { char buf[TD_TIME_STR_LEN] = {0}; - time_t t = taosTime(NULL); + time_t t; + int32_t code = taosTime(&t); + if (code != 0) { + return code; + } struct tm tmInfo; if (taosLocalTime(&t, &tmInfo, buf, sizeof(buf)) != NULL) { (void)strftime(buf, sizeof(buf), "%z", &tmInfo); @@ -196,7 +200,7 @@ static int32_t addTimezoneParam(SNodeList* pList) { int32_t len = (int32_t)strlen(buf); SValueNode* pVal = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pVal); + code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pVal); if (pVal == NULL) { return code; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 4b91f01a8c..750621bf66 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -246,7 +246,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, E return code; } -static int parseTimestampOrInterval(const char** end, SToken* pToken, int16_t timePrec, int64_t* ts, int64_t* interval, +static int32_t parseTimestampOrInterval(const char** end, SToken* pToken, int16_t timePrec, int64_t* ts, int64_t* interval, SMsgBuf* pMsgBuf, bool* isTs) { if (pToken->type == TK_NOW) { *isTs = true; diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index d4d9936154..60339fc646 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -81,6 +81,7 @@ static const char *am_pm[2] = {"AM", "PM"}; #endif char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { + if (!buf || !fmt || !tm) return NULL; #ifdef WINDOWS char c; const char *bp; @@ -345,6 +346,9 @@ char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { } int32_t taosGetTimeOfDay(struct timeval *tv) { + if (tv == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; #ifdef WINDOWS LARGE_INTEGER t; @@ -365,12 +369,15 @@ int32_t taosGetTimeOfDay(struct timeval *tv) { #endif } -time_t taosTime(time_t *t) { +int32_t taosTime(time_t *t) { + if (t == NULL) { + return TSDB_CODE_INVALID_PARA; + } time_t r = time(t); if (r == (time_t)-1) { - terrno = TAOS_SYSTEM_ERROR(errno); + return TAOS_SYSTEM_ERROR(errno); } - return r; + return 0; } /* diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 76d0139521..f70b145dbc 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -154,16 +154,26 @@ static int32_t taosStartLog() { return 0; } -static void getDay(char *buf, int32_t bufSize) { - time_t t = taosTime(NULL); +static int32_t getDay(char *buf, int32_t bufSize) { + time_t t; + int32_t code = taosTime(&t); + if(code != 0) { + return code; + } struct tm tmInfo; if (taosLocalTime(&t, &tmInfo, buf, bufSize) != NULL) { TAOS_UNUSED(strftime(buf, bufSize, "%Y-%m-%d", &tmInfo)); } + return 0; } static int64_t getTimestampToday() { - time_t t = taosTime(NULL); + time_t t; + int32_t code = taosTime(&t); + if (code != 0) { + uError("failed to get time, reason:%s", tstrerror(code)); + return 0; + } struct tm tm; if (taosLocalTime(&t, &tm, NULL, 0) == NULL) { return 0; @@ -203,7 +213,11 @@ int32_t taosInitSlowLog() { char name[PATH_MAX + TD_TIME_STR_LEN] = {0}; char day[TD_TIME_STR_LEN] = {0}; - getDay(day, sizeof(day)); + int32_t code = getDay(day, sizeof(day)); + if (code != 0) { + (void)printf("failed to get day, reason:%s\n", tstrerror(code)); + return code; + } (void)snprintf(name, PATH_MAX + TD_TIME_STR_LEN, "%s.%s", tsLogObj.slowLogName, day); tsLogObj.timestampToday = getTimestampToday(); @@ -434,7 +448,12 @@ static void taosOpenNewSlowLogFile() { atomic_store_32(&tsLogObj.slowHandle->lock, 0); char day[TD_TIME_STR_LEN] = {0}; - getDay(day, sizeof(day)); + int32_t code = getDay(day, sizeof(day)); + if (code != 0) { + uError("failed to get day, reason:%s", tstrerror(code)); + (void)taosThreadMutexUnlock(&tsLogObj.logMutex); + return; + } TdFilePtr pFile = NULL; char name[PATH_MAX + TD_TIME_STR_LEN] = {0}; (void)snprintf(name, PATH_MAX + TD_TIME_STR_LEN, "%s.%s", tsLogObj.slowLogName, day); From a6c7a7c1defc980a59d6cd346eb7f4f9246800b8 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 7 Nov 2024 14:15:22 +0800 Subject: [PATCH 09/44] enh: adjust the num of ins_columns --- tests/system-test/0-others/information_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 01e416bb26..aa548d4e59 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -222,7 +222,7 @@ class TDTestCase: tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'") tdLog.info(len(tdSql.queryResult)) - tdSql.checkEqual(True, len(tdSql.queryResult) in range(280, 281)) + tdSql.checkEqual(True, len(tdSql.queryResult) in range(281, 282)) tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") tdSql.checkEqual(56, len(tdSql.queryResult)) From 7af56a31202d070e9e0b5574fdfce23b4bdae3f0 Mon Sep 17 00:00:00 2001 From: Pengrongkun Date: Thu, 7 Nov 2024 10:27:52 +0800 Subject: [PATCH 10/44] Add TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR, Clear error message --- include/util/taoserror.h | 1 + source/client/src/clientMain.c | 12 ++++++------ source/util/src/terror.c | 1 + source/util/test/errorCodeTable.ini | 1 + 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 9a8b39b84c..2c811495fd 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -208,6 +208,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_TSC_COMPRESS_PARAM_ERROR TAOS_DEF_ERROR_CODE(0, 0X0233) #define TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR TAOS_DEF_ERROR_CODE(0, 0X0234) #define TSDB_CODE_TSC_FAIL_GENERATE_JSON TAOS_DEF_ERROR_CODE(0, 0X0235) +#define TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR TAOS_DEF_ERROR_CODE(0, 0X0236) #define TSDB_CODE_TSC_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0X02FF) // mnode-common diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 4a719373f2..9f6be8e45c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -1803,7 +1803,7 @@ int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { if (bind->num > 1) { tscError("invalid bind number %d for %s", bind->num, __FUNCTION__); - terrno = TSDB_CODE_INVALID_PARA; + terrno = TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR; return terrno; } @@ -1819,7 +1819,7 @@ int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { if (bind->num <= 0 || bind->num > INT16_MAX) { tscError("invalid bind num %d", bind->num); - terrno = TSDB_CODE_INVALID_PARA; + terrno = TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR; return terrno; } @@ -1831,7 +1831,7 @@ int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { } if (0 == insert && bind->num > 1) { tscError("only one row data allowed for query"); - terrno = TSDB_CODE_INVALID_PARA; + terrno = TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR; return terrno; } @@ -1859,7 +1859,7 @@ int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, in } if (0 == insert && bind->num > 1) { tscError("only one row data allowed for query"); - terrno = TSDB_CODE_INVALID_PARA; + terrno = TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR; return terrno; } @@ -2019,7 +2019,7 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col if (bind->num <= 0 || bind->num > INT16_MAX) { tscError("invalid bind num %d", bind->num); - terrno = TSDB_CODE_INVALID_PARA; + terrno = TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR; return terrno; } @@ -2027,7 +2027,7 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col (void)stmtIsInsert2(stmt, &insert); if (0 == insert && bind->num > 1) { tscError("only one row data allowed for query"); - terrno = TSDB_CODE_INVALID_PARA; + terrno = TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR; return terrno; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index d660edd0b8..a5fa9eeb3a 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -164,6 +164,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ENCODE_PARAM_NULL, "Not found compress pa TAOS_DEFINE_ERROR(TSDB_CODE_TSC_COMPRESS_PARAM_ERROR, "Invalid compress param") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR, "Invalid compress level param") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_FAIL_GENERATE_JSON, "failed to generate JSON") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR, "bind number out of range or not match") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INTERNAL_ERROR, "Internal error") diff --git a/source/util/test/errorCodeTable.ini b/source/util/test/errorCodeTable.ini index 33c9d77c5e..e837954a0b 100644 --- a/source/util/test/errorCodeTable.ini +++ b/source/util/test/errorCodeTable.ini @@ -97,6 +97,7 @@ TSDB_CODE_TSC_ENCODE_PARAM_ERROR = 0x80000231 TSDB_CODE_TSC_ENCODE_PARAM_NULL = 0x80000232 TSDB_CODE_TSC_COMPRESS_PARAM_ERROR = 0x80000233 TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR = 0x80000234 +TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR = 0x80000236 TSDB_CODE_TSC_INTERNAL_ERROR = 0x800002FF TSDB_CODE_MND_REQ_REJECTED = 0x80000300 TSDB_CODE_MND_NO_RIGHTS = 0x80000303 From 9473668dd4d7597ff22c506749c7dfefd1e5882a Mon Sep 17 00:00:00 2001 From: haoranchen Date: Thu, 7 Nov 2024 15:46:57 +0800 Subject: [PATCH 11/44] Update test_server.py --- packaging/smokeTest/test_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/smokeTest/test_server.py b/packaging/smokeTest/test_server.py index 768cd2a962..36d86357a3 100644 --- a/packaging/smokeTest/test_server.py +++ b/packaging/smokeTest/test_server.py @@ -40,7 +40,7 @@ def get_config(request): def setup_module(get_config): def run_cmd(command): print("CMD:", command) - result = subprocess.run(command, capture_output=True, text=False, shell=False) + result = subprocess.run(command, capture_output=True, text=True, shell=True) print("STDOUT:", result.stdout) print("STDERR:", result.stderr) print("Return Code:", result.returncode) From 5c01cff369a7882818cd8c44d3bd56b04f24c640 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 7 Nov 2024 16:19:57 +0800 Subject: [PATCH 12/44] enh: param --- source/common/src/tcol.c | 4 ++++ source/libs/command/src/command.c | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/source/common/src/tcol.c b/source/common/src/tcol.c index 923aab12ca..a23385aba0 100644 --- a/source/common/src/tcol.c +++ b/source/common/src/tcol.c @@ -166,6 +166,7 @@ const char* columnCompressStr(uint16_t type) { } uint8_t columnLevelVal(const char* level) { + if (level == NULL) return TSDB_COLVAL_LEVEL_NOCHANGE; uint8_t l = TSDB_COLVAL_LEVEL_MEDIUM; if (0 == strcmp(level, "h") || 0 == strcmp(level, TSDB_COLUMN_LEVEL_HIGH)) { l = TSDB_COLVAL_LEVEL_HIGH; @@ -180,6 +181,7 @@ uint8_t columnLevelVal(const char* level) { } uint16_t columnCompressVal(const char* compress) { + if (compress == NULL) return TSDB_COLVAL_COMPRESS_NOCHANGE; uint16_t c = TSDB_COLVAL_COMPRESS_LZ4; if (0 == strcmp(compress, TSDB_COLUMN_COMPRESS_LZ4)) { c = TSDB_COLVAL_COMPRESS_LZ4; @@ -200,6 +202,7 @@ uint16_t columnCompressVal(const char* compress) { } uint8_t columnEncodeVal(const char* encode) { + if (encode == NULL) return TSDB_COLVAL_ENCODE_NOCHANGE; uint8_t e = TSDB_COLVAL_ENCODE_SIMPLE8B; if (0 == strcmp(encode, TSDB_COLUMN_ENCODE_SIMPLE8B)) { e = TSDB_COLVAL_ENCODE_SIMPLE8B; @@ -311,6 +314,7 @@ void setColLevel(uint32_t* compress, uint8_t level) { int32_t setColCompressByOption(uint8_t type, uint8_t encode, uint16_t compressType, uint8_t level, bool check, uint32_t* compress) { + if(compress == NULL) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; if (check && !validColEncode(type, encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; setColEncode(compress, encode); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 6272ac7049..5afdf87afb 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -35,6 +35,9 @@ extern SConfig* tsCfg; static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRetrieveTableRsp** pRsp) { + if (NULL == pBlock || NULL == pRsp) { + return TSDB_CODE_INVALID_PARA; + } size_t dataEncodeBufSize = blockGetEncodeSize(pBlock); size_t rspSize = sizeof(SRetrieveTableRsp) + dataEncodeBufSize + PAYLOAD_PREFIX_LEN; *pRsp = taosMemoryCalloc(1, rspSize); @@ -216,6 +219,9 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp, int8_t biMode) { SDescribeStmt* pDesc = (SDescribeStmt*)pStmt; + if (NULL == pDesc || NULL == pDesc->pMeta) { + return TSDB_CODE_INVALID_PARA; + } int32_t numOfRows = TABLE_TOTAL_COL_NUM(pDesc->pMeta); SSDataBlock* pBlock = NULL; @@ -505,7 +511,7 @@ static int32_t buildCreateViewResultDataBlock(SSDataBlock** pOutput) { return code; } -void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { +static void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfColumns; ++i) { SSchema* pSchema = pCfg->pSchemas + i; #define LTYPE_LEN (32 + 60) // 60 byte for compress info @@ -539,7 +545,7 @@ void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { } } -void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) { +static void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfTags; ++i) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; char type[32]; @@ -558,7 +564,7 @@ void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) { } } -void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) { +static void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfTags; ++i) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), @@ -566,7 +572,7 @@ void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) { } } -int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { +static int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { int32_t code = TSDB_CODE_SUCCESS; SArray* pTagVals = NULL; STag* pTag = (STag*)pCfg->pTags; @@ -643,7 +649,7 @@ _exit: return code; } -void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* pCfg) { +static void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* pCfg) { if (pCfg->commentLen > 0) { *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), " COMMENT '%s'", pCfg->pComment); @@ -997,7 +1003,7 @@ static int32_t createSelectResultDataBlock(SNodeList* pProjects, SSDataBlock** p return code; } -int32_t buildSelectResultDataBlock(SNodeList* pProjects, SSDataBlock* pBlock) { +static int32_t buildSelectResultDataBlock(SNodeList* pProjects, SSDataBlock* pBlock) { QRY_ERR_RET(blockDataEnsureCapacity(pBlock, 1)); int32_t index = 0; From fa62b95b1fb46b827995c5bc9528c8ccacd296ec Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 7 Nov 2024 16:20:33 +0800 Subject: [PATCH 13/44] test: add smoke test scripts --- packaging/delete_ref_lock.py | 59 +++++ packaging/smoke_test/assets/style.css | 319 ++++++++++++++++++++++++ packaging/smoke_test/pytest_require.txt | 16 ++ 3 files changed, 394 insertions(+) create mode 100644 packaging/delete_ref_lock.py create mode 100644 packaging/smoke_test/assets/style.css create mode 100644 packaging/smoke_test/pytest_require.txt diff --git a/packaging/delete_ref_lock.py b/packaging/delete_ref_lock.py new file mode 100644 index 0000000000..cf0e4cdd05 --- /dev/null +++ b/packaging/delete_ref_lock.py @@ -0,0 +1,59 @@ +import subprocess +import re + +# 执行 git fetch 命令并捕获输出 +def git_fetch(): + result = subprocess.run(['git', 'fetch'], capture_output=True, text=True) + return result + +# 解析分支名称 +def parse_branch_name_type1(error_output): + # 使用正则表达式匹配 'is at' 前的分支名称 + match = re.search(r"error: cannot lock ref '(refs/remotes/origin/[^']+)': is at", error_output) + if match: + return match.group(1) + return None + +# 解析第二种错误中的分支名称 +def parse_branch_name_type2(error_output): + # 使用正则表达式匹配 'exists' 前的第一个引号内的分支名称 + match = re.search(r"'(refs/remotes/origin/[^']+)' exists;", error_output) + if match: + return match.group(1) + return None + +# 执行 git update-ref -d 命令 +def git_update_ref(branch_name): + if branch_name: + subprocess.run(['git', 'update-ref', '-d', f'{branch_name}'], check=True) + +# 解析错误类型并执行相应的修复操作 +def handle_error(error_output): + # 错误类型1：本地引用的提交ID与远程不一致 + if "is at" in error_output and "but expected" in error_output: + branch_name = parse_branch_name_type1(error_output) + if branch_name: + print(f"Detected error type 1, attempting to delete ref for branch: {branch_name}") + git_update_ref(branch_name) + else: + print("Error parsing branch name for type 1.") + # 错误类型2：尝试创建新的远程引用时，本地已经存在同名的引用 + elif "exists; cannot create" in error_output: + branch_name = parse_branch_name_type2(error_output) + if branch_name: + print(f"Detected error type 2, attempting to delete ref for branch: {branch_name}") + git_update_ref(branch_name) + else: + print("Error parsing branch name for type 2.") + +# 主函数 +def main(): + fetch_result = git_fetch() + if fetch_result.returncode != 0: # 如果 git fetch 命令失败 + error_output = fetch_result.stderr + handle_error(error_output) + else: + print("Git fetch successful.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/packaging/smoke_test/assets/style.css b/packaging/smoke_test/assets/style.css new file mode 100644 index 0000000000..c89d42818c --- /dev/null +++ b/packaging/smoke_test/assets/style.css @@ -0,0 +1,319 @@ +body { + font-family: Helvetica, Arial, sans-serif; + font-size: 12px; + /* do not increase min-width as some may use split screens */ + min-width: 800px; + color: #999; + } + + h1 { + font-size: 24px; + color: black; + } + + h2 { + font-size: 16px; + color: black; + } + + p { + color: black; + } + + a { + color: #999; + } + + table { + border-collapse: collapse; + } + + /****************************** + * SUMMARY INFORMATION + ******************************/ + #environment td { + padding: 5px; + border: 1px solid #e6e6e6; + vertical-align: top; + } + #environment tr:nth-child(odd) { + background-color: #f6f6f6; + } + #environment ul { + margin: 0; + padding: 0 20px; + } + + /****************************** + * TEST RESULT COLORS + ******************************/ + span.passed, + .passed .col-result { + color: green; + } + + span.skipped, + span.xfailed, + span.rerun, + .skipped .col-result, + .xfailed .col-result, + .rerun .col-result { + color: orange; + } + + span.error, + span.failed, + span.xpassed, + .error .col-result, + .failed .col-result, + .xpassed .col-result { + color: red; + } + + .col-links__extra { + margin-right: 3px; + } + + /****************************** + * RESULTS TABLE + * + * 1. Table Layout + * 2. Extra + * 3. Sorting items + * + ******************************/ + /*------------------ + * 1. Table Layout + *------------------*/ + #results-table { + border: 1px solid #e6e6e6; + color: #999; + font-size: 12px; + width: 100%; + } + #results-table th, + #results-table td { + padding: 5px; + border: 1px solid #e6e6e6; + text-align: left; + } + #results-table th { + font-weight: bold; + } + + /*------------------ + * 2. Extra + *------------------*/ + .logwrapper { + max-height: 230px; + overflow-y: scroll; + background-color: #e6e6e6; + } + .logwrapper.expanded { + max-height: none; + } + .logwrapper.expanded .logexpander:after { + content: "collapse [-]"; + } + .logwrapper .logexpander { + z-index: 1; + position: sticky; + top: 10px; + width: max-content; + border: 1px solid; + border-radius: 3px; + padding: 5px 7px; + margin: 10px 0 10px calc(100% - 80px); + cursor: pointer; + background-color: #e6e6e6; + } + .logwrapper .logexpander:after { + content: "expand [+]"; + } + .logwrapper .logexpander:hover { + color: #000; + border-color: #000; + } + .logwrapper .log { + min-height: 40px; + position: relative; + top: -50px; + height: calc(100% + 50px); + border: 1px solid #e6e6e6; + color: black; + display: block; + font-family: "Courier New", Courier, monospace; + padding: 5px; + padding-right: 80px; + white-space: pre-wrap; + } + + div.media { + border: 1px solid #e6e6e6; + float: right; + height: 240px; + margin: 0 5px; + overflow: hidden; + width: 320px; + } + + .media-container { + display: grid; + grid-template-columns: 25px auto 25px; + align-items: center; + flex: 1 1; + overflow: hidden; + height: 200px; + } + + .media-container--fullscreen { + grid-template-columns: 0px auto 0px; + } + + .media-container__nav--right, + .media-container__nav--left { + text-align: center; + cursor: pointer; + } + + .media-container__viewport { + cursor: pointer; + text-align: center; + height: inherit; + } + .media-container__viewport img, + .media-container__viewport video { + object-fit: cover; + width: 100%; + max-height: 100%; + } + + .media__name, + .media__counter { + display: flex; + flex-direction: row; + justify-content: space-around; + flex: 0 0 25px; + align-items: center; + } + + .collapsible td:not(.col-links) { + cursor: pointer; + } + .collapsible td:not(.col-links):hover::after { + color: #bbb; + font-style: italic; + cursor: pointer; + } + + .col-result { + width: 130px; + } + .col-result:hover::after { + content: " (hide details)"; + } + + .col-result.collapsed:hover::after { + content: " (show details)"; + } + + #environment-header h2:hover::after { + content: " (hide details)"; + color: #bbb; + font-style: italic; + cursor: pointer; + font-size: 12px; + } + + #environment-header.collapsed h2:hover::after { + content: " (show details)"; + color: #bbb; + font-style: italic; + cursor: pointer; + font-size: 12px; + } + + /*------------------ + * 3. Sorting items + *------------------*/ + .sortable { + cursor: pointer; + } + .sortable.desc:after { + content: " "; + position: relative; + left: 5px; + bottom: -12.5px; + border: 10px solid #4caf50; + border-bottom: 0; + border-left-color: transparent; + border-right-color: transparent; + } + .sortable.asc:after { + content: " "; + position: relative; + left: 5px; + bottom: 12.5px; + border: 10px solid #4caf50; + border-top: 0; + border-left-color: transparent; + border-right-color: transparent; + } + + .hidden, .summary__reload__button.hidden { + display: none; + } + + .summary__data { + flex: 0 0 550px; + } + .summary__reload { + flex: 1 1; + display: flex; + justify-content: center; + } + .summary__reload__button { + flex: 0 0 300px; + display: flex; + color: white; + font-weight: bold; + background-color: #4caf50; + text-align: center; + justify-content: center; + align-items: center; + border-radius: 3px; + cursor: pointer; + } + .summary__reload__button:hover { + background-color: #46a049; + } + .summary__spacer { + flex: 0 0 550px; + } + + .controls { + display: flex; + justify-content: space-between; + } + + .filters, + .collapse { + display: flex; + align-items: center; + } + .filters button, + .collapse button { + color: #999; + border: none; + background: none; + cursor: pointer; + text-decoration: underline; + } + .filters button:hover, + .collapse button:hover { + color: #ccc; + } + + .filter__label { + margin-right: 10px; + } \ No newline at end of file diff --git a/packaging/smoke_test/pytest_require.txt b/packaging/smoke_test/pytest_require.txt new file mode 100644 index 0000000000..eadbe92470 --- /dev/null +++ b/packaging/smoke_test/pytest_require.txt @@ -0,0 +1,16 @@ +pytest-html +pytest-json-report +pytest-timeout +taospy +numpy +fabric2 +psutil +pandas +toml +distro +requests +pexpect +faker +pyopenssl +taos-ws-py +taospy \ No newline at end of file From 446122ad9650258832e1a212232dc961152e7fc2 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 7 Nov 2024 16:22:30 +0800 Subject: [PATCH 14/44] test: add smoke test scripts --- packaging/{smoke_test => smokeTest}/assets/style.css | 0 packaging/smoke_test/pytest_require.txt | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) rename packaging/{smoke_test => smokeTest}/assets/style.css (100%) diff --git a/packaging/smoke_test/assets/style.css b/packaging/smokeTest/assets/style.css similarity index 100% rename from packaging/smoke_test/assets/style.css rename to packaging/smokeTest/assets/style.css diff --git a/packaging/smoke_test/pytest_require.txt b/packaging/smoke_test/pytest_require.txt index eadbe92470..34019c6e8a 100644 --- a/packaging/smoke_test/pytest_require.txt +++ b/packaging/smoke_test/pytest_require.txt @@ -13,4 +13,5 @@ pexpect faker pyopenssl taos-ws-py -taospy \ No newline at end of file +taospy +tzlocal \ No newline at end of file From 234846b5436dd00f447f0cfd1ea182253b3eef4e Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 7 Nov 2024 16:24:54 +0800 Subject: [PATCH 15/44] test: add smoke test scripts --- .gitignore | 2 +- packaging/{smoke_test => smokeTest}/pytest_require.txt | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename packaging/{smoke_test => smokeTest}/pytest_require.txt (100%) diff --git a/.gitignore b/.gitignore index 7f2382db87..8f461f2b02 100644 --- a/.gitignore +++ b/.gitignore @@ -161,4 +161,4 @@ version.h geos_c.h source/libs/parser/src/sql.c include/common/ttokenauto.h -!packaging/smoke_test/pytest_require.txt +!packaging/smokeTest/pytest_require.txt diff --git a/packaging/smoke_test/pytest_require.txt b/packaging/smokeTest/pytest_require.txt similarity index 100% rename from packaging/smoke_test/pytest_require.txt rename to packaging/smokeTest/pytest_require.txt From daedcc53fb2b84b5a9acafffb0486cb3b84dd952 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Nov 2024 16:28:47 +0800 Subject: [PATCH 16/44] refactor: return the error code and do some refactor. --- .../libs/executor/src/anomalywindowoperator.c | 17 ++++++++++++----- source/util/src/tanal.c | 3 +++ source/util/src/terror.c | 4 ++-- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/anomalywindowoperator.c b/source/libs/executor/src/anomalywindowoperator.c index d03e527c2b..6adfbc6c77 100644 --- a/source/libs/executor/src/anomalywindowoperator.c +++ b/source/libs/executor/src/anomalywindowoperator.c @@ -55,7 +55,7 @@ typedef struct { static void anomalyDestroyOperatorInfo(void* param); static int32_t anomalyAggregateNext(SOperatorInfo* pOperator, SSDataBlock** ppRes); -static void anomalyAggregateBlocks(SOperatorInfo* pOperator); +static int32_t anomalyAggregateBlocks(SOperatorInfo* pOperator); static int32_t anomalyCacheBlock(SAnomalyWindowOperatorInfo* pInfo, SSDataBlock* pBlock); int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo, @@ -78,6 +78,7 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p code = TSDB_CODE_ANAL_ALGO_NOT_FOUND; goto _error; } + if (taosAnalGetAlgoUrl(pInfo->algoName, ANAL_ALGO_TYPE_ANOMALY_DETECT, pInfo->algoUrl, sizeof(pInfo->algoUrl)) != 0) { qError("failed to get anomaly_window algorithm url from %s", pInfo->algoName); code = TSDB_CODE_ANAL_ALGO_NOT_LOAD; @@ -198,7 +199,9 @@ static int32_t anomalyAggregateNext(SOperatorInfo* pOperator, SSDataBlock** ppRe QUERY_CHECK_CODE(code, lino, _end); } else { qDebug("group:%" PRId64 ", read finish for new group coming, blocks:%d", pSupp->groupId, numOfBlocks); - anomalyAggregateBlocks(pOperator); + code = anomalyAggregateBlocks(pOperator); + QUERY_CHECK_CODE(code, lino, _end); + pSupp->groupId = pBlock->info.id.groupId; numOfBlocks = 1; pSupp->cachedRows = pBlock->info.rows; @@ -217,7 +220,7 @@ static int32_t anomalyAggregateNext(SOperatorInfo* pOperator, SSDataBlock** ppRe if (numOfBlocks > 0) { qDebug("group:%" PRId64 ", read finish, blocks:%d", pInfo->anomalySup.groupId, numOfBlocks); - anomalyAggregateBlocks(pOperator); + code = anomalyAggregateBlocks(pOperator); } int64_t cost = taosGetTimestampUs() - st; @@ -229,6 +232,7 @@ _end: pTaskInfo->code = code; T_LONG_JMP(pTaskInfo->env, code); } + (*ppRes) = (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes; return code; } @@ -338,8 +342,8 @@ static int32_t anomalyAnalysisWindow(SOperatorInfo* pOperator) { SAnalBuf analBuf = {.bufType = ANAL_BUF_TYPE_JSON}; char dataBuf[64] = {0}; int32_t code = 0; + int64_t ts = 0; - int64_t ts = 0; // int64_t ts = taosGetTimestampMs(); snprintf(analBuf.fileName, sizeof(analBuf.fileName), "%s/tdengine-anomaly-%" PRId64 "-%" PRId64, tsTempDir, ts, pSupp->groupId); @@ -431,6 +435,7 @@ _OVER: if (code != 0) { qError("failed to analysis window since %s", tstrerror(code)); } + taosAnalBufDestroy(&analBuf); if (pJson != NULL) tjsonDelete(pJson); return code; @@ -473,7 +478,7 @@ static int32_t anomalyBuildResult(SOperatorInfo* pOperator) { return code; } -static void anomalyAggregateBlocks(SOperatorInfo* pOperator) { +static int32_t anomalyAggregateBlocks(SOperatorInfo* pOperator) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SAnomalyWindowOperatorInfo* pInfo = pOperator->info; @@ -623,6 +628,8 @@ _OVER: pSupp->curWin.ekey = 0; pSupp->curWin.skey = 0; pSupp->curWinIndex = 0; + + return code; } #else diff --git a/source/util/src/tanal.c b/source/util/src/tanal.c index 92eee28ba8..bb46a48c42 100644 --- a/source/util/src/tanal.c +++ b/source/util/src/tanal.c @@ -167,6 +167,8 @@ int32_t taosAnalGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, char name[TSDB_ANAL_ALGO_KEY_LEN] = {0}; int32_t nameLen = 1 + tsnprintf(name, sizeof(name) - 1, "%d:%s", type, algoName); + char *unused = strntolower(name, name, nameLen); + if (taosThreadMutexLock(&tsAlgos.lock) == 0) { SAnalUrl *pUrl = taosHashAcquire(tsAlgos.hash, name, nameLen); if (pUrl != NULL) { @@ -178,6 +180,7 @@ int32_t taosAnalGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, code = terrno; uError("algo:%s, type:%s, url not found", algoName, taosAnalAlgoStr(type)); } + if (taosThreadMutexUnlock(&tsAlgos.lock) != 0) { uError("failed to unlock hash"); return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index d660edd0b8..ce209cc718 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -360,8 +360,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_ANODE_TOO_MANY_ALGO, "Anode too many algori TAOS_DEFINE_ERROR(TSDB_CODE_MND_ANODE_TOO_LONG_ALGO_NAME, "Anode too long algorithm name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ANODE_TOO_MANY_ALGO_TYPE, "Anode too many algorithm type") -TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_URL_RSP_IS_NULL, "Analysis url response is NULL") -TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_URL_CANT_ACCESS, "Analysis url can't access") +TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_URL_RSP_IS_NULL, "Analysis service response is NULL") +TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_URL_CANT_ACCESS, "Analysis service can't access") TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_ALGO_NOT_FOUND, "Analysis algorithm not found") TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_ALGO_NOT_LOAD, "Analysis algorithm not loaded") TAOS_DEFINE_ERROR(TSDB_CODE_ANAL_BUF_INVALID_TYPE, "Analysis invalid buffer type") From a8f7f153d92ce3ddb4dba07af57d7ec45fcb1bad Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Nov 2024 16:44:43 +0800 Subject: [PATCH 17/44] refactor: do some internal refacrtor. --- include/common/{tanal.h => tanalytics.h} | 10 +++---- source/common/src/tmsg.c | 12 ++++---- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 4 +-- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 4 +-- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 2 +- source/dnode/mnode/impl/CMakeLists.txt | 2 +- source/dnode/mnode/impl/src/mndAnode.c | 10 +++---- source/libs/executor/CMakeLists.txt | 2 +- .../libs/executor/src/anomalywindowoperator.c | 4 +-- source/libs/executor/src/forecastoperator.c | 4 +-- source/libs/function/src/builtins.c | 2 +- source/libs/function/src/builtinsimpl.c | 2 +- source/libs/parser/src/parTranslater.c | 2 +- source/util/CMakeLists.txt | 2 +- source/util/src/{tanal.c => tanalytics.c} | 29 +++++++++---------- 16 files changed, 46 insertions(+), 47 deletions(-) rename include/common/{tanal.h => tanalytics.h} (96%) rename source/util/src/{tanal.c => tanalytics.c} (97%) diff --git a/include/common/tanal.h b/include/common/tanalytics.h similarity index 96% rename from include/common/tanal.h rename to include/common/tanalytics.h index 69d110d161..85eb963129 100644 --- a/include/common/tanal.h +++ b/include/common/tanalytics.h @@ -36,7 +36,7 @@ typedef struct { int32_t anode; int32_t urlLen; char *url; -} SAnalUrl; +} SAnalyticsUrl; typedef enum { ANAL_BUF_TYPE_JSON = 0, @@ -53,18 +53,18 @@ typedef struct { TdFilePtr filePtr; char fileName[TSDB_FILENAME_LEN + 10]; int64_t numOfRows; -} SAnalColBuf; +} SAnalyticsColBuf; typedef struct { EAnalBufType bufType; TdFilePtr filePtr; char fileName[TSDB_FILENAME_LEN]; int32_t numOfCols; - SAnalColBuf *pCols; + SAnalyticsColBuf *pCols; } SAnalBuf; -int32_t taosAnalInit(); -void taosAnalCleanup(); +int32_t taosAnalyticsInit(); +void taosAnalyticsCleanup(); SJson *taosAnalSendReqRetJson(const char *url, EAnalHttpType type, SAnalBuf *pBuf); int32_t taosAnalGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, int32_t urlLen); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 458badc764..6d1699b911 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -40,7 +40,7 @@ #define TD_MSG_RANGE_CODE_ #include "tmsgdef.h" -#include "tanal.h" +#include "tanalytics.h" #include "tcol.h" #include "tlog.h" @@ -2166,7 +2166,7 @@ int32_t tSerializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnalAl int32_t numOfAlgos = 0; void *pIter = taosHashIterate(pRsp->hash, NULL); while (pIter != NULL) { - SAnalUrl *pUrl = pIter; + SAnalyticsUrl *pUrl = pIter; size_t nameLen = 0; const char *name = taosHashGetKey(pIter, &nameLen); if (nameLen > 0 && nameLen <= TSDB_ANAL_ALGO_KEY_LEN && pUrl->urlLen > 0) { @@ -2181,7 +2181,7 @@ int32_t tSerializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnalAl pIter = taosHashIterate(pRsp->hash, NULL); while (pIter != NULL) { - SAnalUrl *pUrl = pIter; + SAnalyticsUrl *pUrl = pIter; size_t nameLen = 0; const char *name = taosHashGetKey(pIter, &nameLen); if (nameLen > 0 && pUrl->urlLen > 0) { @@ -2225,7 +2225,7 @@ int32_t tDeserializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnal int32_t nameLen; int32_t type; char name[TSDB_ANAL_ALGO_KEY_LEN]; - SAnalUrl url = {0}; + SAnalyticsUrl url = {0}; TAOS_CHECK_EXIT(tStartDecode(&decoder)); TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pRsp->ver)); @@ -2245,7 +2245,7 @@ int32_t tDeserializeRetrieveAnalAlgoRsp(void *buf, int32_t bufLen, SRetrieveAnal TAOS_CHECK_EXIT(tDecodeBinaryAlloc(&decoder, (void **)&url.url, NULL) < 0); } - TAOS_CHECK_EXIT(taosHashPut(pRsp->hash, name, nameLen, &url, sizeof(SAnalUrl))); + TAOS_CHECK_EXIT(taosHashPut(pRsp->hash, name, nameLen, &url, sizeof(SAnalyticsUrl))); } tEndDecode(&decoder); @@ -2258,7 +2258,7 @@ _exit: void tFreeRetrieveAnalAlgoRsp(SRetrieveAnalAlgoRsp *pRsp) { void *pIter = taosHashIterate(pRsp->hash, NULL); while (pIter != NULL) { - SAnalUrl *pUrl = (SAnalUrl *)pIter; + SAnalyticsUrl *pUrl = (SAnalyticsUrl *)pIter; taosMemoryFree(pUrl->url); pIter = taosHashIterate(pRsp->hash, pIter); } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index d6b792ca74..78cc35a62c 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -18,7 +18,7 @@ #include "dmInt.h" #include "monitor.h" #include "systable.h" -#include "tanal.h" +#include "tanalytics.h" #include "tchecksum.h" extern SConfig *tsCfg; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 04b4e9101c..fb7d891c67 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "dmInt.h" #include "libs/function/tudf.h" -#include "tanal.h" +#include "tanalytics.h" static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) { int32_t code = 0; @@ -85,7 +85,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { dError("failed to start udfd since %s", tstrerror(code)); } - if ((code = taosAnalInit()) != 0) { + if ((code = taosAnalyticsInit()) != 0) { dError("failed to init analysis env since %s", tstrerror(code)); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 694cc52d64..6d4ebe424a 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -21,7 +21,7 @@ #include "tgrant.h" #include "tcompare.h" #include "tcs.h" -#include "tanal.h" +#include "tanalytics.h" // clang-format on #define DM_INIT_AUDIT() \ @@ -209,7 +209,7 @@ void dmCleanup() { dError("failed to close udfc"); } udfStopUdfd(); - taosAnalCleanup(); + taosAnalyticsCleanup(); taosStopCacheRefreshWorker(); (void)dmDiskClose(); DestroyRegexCache(); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 5a276de251..61543e619e 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "dmMgmt.h" #include "qworker.h" -#include "tanal.h" +#include "tanalytics.h" #include "tversion.h" static inline void dmSendRsp(SRpcMsg *pMsg) { diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 8a390948ae..ad36d8c8ae 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -18,7 +18,7 @@ if(TD_ENTERPRISE) endif() if(${BUILD_WITH_ANALYSIS}) - add_definitions(-DUSE_ANAL) + add_definitions(-DUSE_ANALYTICS) endif() endif() diff --git a/source/dnode/mnode/impl/src/mndAnode.c b/source/dnode/mnode/impl/src/mndAnode.c index 17e3e84c81..87bfe9f7af 100644 --- a/source/dnode/mnode/impl/src/mndAnode.c +++ b/source/dnode/mnode/impl/src/mndAnode.c @@ -21,10 +21,10 @@ #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" -#include "tanal.h" +#include "tanalytics.h" #include "tjson.h" -#ifdef USE_ANAL +#ifdef USE_ANALYTICS #define TSDB_ANODE_VER_NUMBER 1 #define TSDB_ANODE_RESERVE_SIZE 64 @@ -806,7 +806,7 @@ static int32_t mndProcessAnalAlgoReq(SRpcMsg *pReq) { SSdb *pSdb = pMnode->pSdb; int32_t code = -1; SAnodeObj *pObj = NULL; - SAnalUrl url; + SAnalyticsUrl url; int32_t nameLen; char name[TSDB_ANAL_ALGO_KEY_LEN]; SRetrieveAnalAlgoReq req = {0}; @@ -838,7 +838,7 @@ static int32_t mndProcessAnalAlgoReq(SRpcMsg *pReq) { SAnodeAlgo *algo = taosArrayGet(algos, a); nameLen = 1 + tsnprintf(name, sizeof(name) - 1, "%d:%s", url.type, algo->name); - SAnalUrl *pOldUrl = taosHashAcquire(rsp.hash, name, nameLen); + SAnalyticsUrl *pOldUrl = taosHashAcquire(rsp.hash, name, nameLen); if (pOldUrl == NULL || (pOldUrl != NULL && pOldUrl->anode < url.anode)) { if (pOldUrl != NULL) { taosMemoryFreeClear(pOldUrl->url); @@ -855,7 +855,7 @@ static int32_t mndProcessAnalAlgoReq(SRpcMsg *pReq) { url.urlLen = 1 + tsnprintf(url.url, TSDB_ANAL_ANODE_URL_LEN + TSDB_ANAL_ALGO_TYPE_LEN, "%s/%s", pAnode->url, taosAnalAlgoUrlStr(url.type)); - if (taosHashPut(rsp.hash, name, nameLen, &url, sizeof(SAnalUrl)) != 0) { + if (taosHashPut(rsp.hash, name, nameLen, &url, sizeof(SAnalyticsUrl)) != 0) { taosMemoryFree(url.url); sdbRelease(pSdb, pAnode); goto _OVER; diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 014b538375..9a49076b6b 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -7,7 +7,7 @@ if(${TD_DARWIN}) endif(${TD_DARWIN}) if(${BUILD_WITH_ANALYSIS}) - add_definitions(-DUSE_ANAL) + add_definitions(-DUSE_ANALYTICS) endif() target_link_libraries(executor diff --git a/source/libs/executor/src/anomalywindowoperator.c b/source/libs/executor/src/anomalywindowoperator.c index 6adfbc6c77..94cc5d9129 100644 --- a/source/libs/executor/src/anomalywindowoperator.c +++ b/source/libs/executor/src/anomalywindowoperator.c @@ -19,14 +19,14 @@ #include "functionMgt.h" #include "operator.h" #include "querytask.h" -#include "tanal.h" +#include "tanalytics.h" #include "tcommon.h" #include "tcompare.h" #include "tdatablock.h" #include "tjson.h" #include "ttime.h" -#ifdef USE_ANAL +#ifdef USE_ANALYTICS typedef struct { SArray* blocks; // SSDataBlock* diff --git a/source/libs/executor/src/forecastoperator.c b/source/libs/executor/src/forecastoperator.c index 0afa933ee8..20dc9e28ba 100644 --- a/source/libs/executor/src/forecastoperator.c +++ b/source/libs/executor/src/forecastoperator.c @@ -19,14 +19,14 @@ #include "operator.h" #include "querytask.h" #include "storageapi.h" -#include "tanal.h" +#include "tanalytics.h" #include "tcommon.h" #include "tcompare.h" #include "tdatablock.h" #include "tfill.h" #include "ttime.h" -#ifdef USE_ANAL +#ifdef USE_ANALYTICS typedef struct { char algoName[TSDB_ANAL_ALGO_NAME_LEN]; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 552933dcad..2d68eb9d51 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -19,7 +19,7 @@ #include "geomFunc.h" #include "querynodes.h" #include "scalar.h" -#include "tanal.h" +#include "tanalytics.h" #include "taoserror.h" #include "ttime.h" diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 0aad1501ce..acdac7cbc3 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -19,7 +19,7 @@ #include "functionResInfoInt.h" #include "query.h" #include "querynodes.h" -#include "tanal.h" +#include "tanalytics.h" #include "tcompare.h" #include "tdatablock.h" #include "tdigest.h" diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 02295b34da..99c03c412c 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -24,7 +24,7 @@ #include "parUtil.h" #include "scalar.h" #include "systable.h" -#include "tanal.h" +#include "tanalytics.h" #include "tcol.h" #include "tglobal.h" #include "ttime.h" diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 7f5955f3dd..2633bb3268 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -18,7 +18,7 @@ else() endif(${ASSERT_NOT_CORE}) if(${BUILD_WITH_ANALYSIS}) - add_definitions(-DUSE_ANAL) + add_definitions(-DUSE_ANALYTICS) endif() target_include_directories( diff --git a/source/util/src/tanal.c b/source/util/src/tanalytics.c similarity index 97% rename from source/util/src/tanal.c rename to source/util/src/tanalytics.c index bb46a48c42..99d91700a2 100644 --- a/source/util/src/tanal.c +++ b/source/util/src/tanalytics.c @@ -14,18 +14,17 @@ */ #define _DEFAULT_SOURCE -#include "tanal.h" -#include "tmsg.h" +#include "tanalytics.h" #include "ttypes.h" #include "tutil.h" -#ifdef USE_ANAL +#ifdef USE_ANALYTICS #include #define ANAL_ALGO_SPLIT "," typedef struct { int64_t ver; - SHashObj *hash; // algoname:algotype -> SAnalUrl + SHashObj *hash; // algoname:algotype -> SAnalyticsUrl TdThreadMutex lock; } SAlgoMgmt; @@ -69,7 +68,7 @@ EAnalAlgoType taosAnalAlgoInt(const char *name) { return ANAL_ALGO_TYPE_END; } -int32_t taosAnalInit() { +int32_t taosAnalyticsInit() { if (curl_global_init(CURL_GLOBAL_ALL) != 0) { uError("failed to init curl"); return -1; @@ -94,14 +93,14 @@ int32_t taosAnalInit() { static void taosAnalFreeHash(SHashObj *hash) { void *pIter = taosHashIterate(hash, NULL); while (pIter != NULL) { - SAnalUrl *pUrl = (SAnalUrl *)pIter; + SAnalyticsUrl *pUrl = (SAnalyticsUrl *)pIter; taosMemoryFree(pUrl->url); pIter = taosHashIterate(hash, pIter); } taosHashCleanup(hash); } -void taosAnalCleanup() { +void taosAnalyticsCleanup() { curl_global_cleanup(); if (taosThreadMutexDestroy(&tsAlgos.lock) != 0) { uError("failed to destroy anal lock"); @@ -170,7 +169,7 @@ int32_t taosAnalGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, char *unused = strntolower(name, name, nameLen); if (taosThreadMutexLock(&tsAlgos.lock) == 0) { - SAnalUrl *pUrl = taosHashAcquire(tsAlgos.hash, name, nameLen); + SAnalyticsUrl *pUrl = taosHashAcquire(tsAlgos.hash, name, nameLen); if (pUrl != NULL) { tstrncpy(url, pUrl->url, urlLen); uDebug("algo:%s, type:%s, url:%s", algoName, taosAnalAlgoStr(type), url); @@ -406,7 +405,7 @@ static int32_t tsosAnalJsonBufOpen(SAnalBuf *pBuf, int32_t numOfCols) { return terrno; } - pBuf->pCols = taosMemoryCalloc(numOfCols, sizeof(SAnalColBuf)); + pBuf->pCols = taosMemoryCalloc(numOfCols, sizeof(SAnalyticsColBuf)); if (pBuf->pCols == NULL) return TSDB_CODE_OUT_OF_MEMORY; pBuf->numOfCols = numOfCols; @@ -415,7 +414,7 @@ static int32_t tsosAnalJsonBufOpen(SAnalBuf *pBuf, int32_t numOfCols) { } for (int32_t i = 0; i < numOfCols; ++i) { - SAnalColBuf *pCol = &pBuf->pCols[i]; + SAnalyticsColBuf *pCol = &pBuf->pCols[i]; snprintf(pCol->fileName, sizeof(pCol->fileName), "%s-c%d", pBuf->fileName, i); pCol->filePtr = taosOpenFile(pCol->fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); @@ -549,7 +548,7 @@ static int32_t taosAnalJsonBufWriteDataEnd(SAnalBuf *pBuf) { if (pBuf->bufType == ANAL_BUF_TYPE_JSON_COL) { for (int32_t i = 0; i < pBuf->numOfCols; ++i) { - SAnalColBuf *pCol = &pBuf->pCols[i]; + SAnalyticsColBuf *pCol = &pBuf->pCols[i]; code = taosFsyncFile(pCol->filePtr); if (code != 0) return code; @@ -591,7 +590,7 @@ int32_t taosAnalJsonBufClose(SAnalBuf *pBuf) { if (pBuf->bufType == ANAL_BUF_TYPE_JSON_COL) { for (int32_t i = 0; i < pBuf->numOfCols; ++i) { - SAnalColBuf *pCol = &pBuf->pCols[i]; + SAnalyticsColBuf *pCol = &pBuf->pCols[i]; if (pCol->filePtr != NULL) { code = taosFsyncFile(pCol->filePtr); if (code != 0) return code; @@ -613,7 +612,7 @@ void taosAnalBufDestroy(SAnalBuf *pBuf) { if (pBuf->bufType == ANAL_BUF_TYPE_JSON_COL) { for (int32_t i = 0; i < pBuf->numOfCols; ++i) { - SAnalColBuf *pCol = &pBuf->pCols[i]; + SAnalyticsColBuf *pCol = &pBuf->pCols[i]; if (pCol->fileName[0] != 0) { if (pCol->filePtr != NULL) (void)taosCloseFile(&pCol->filePtr); if (taosRemoveFile(pCol->fileName) != 0) { @@ -729,8 +728,8 @@ static int32_t taosAnalBufGetCont(SAnalBuf *pBuf, char **ppCont, int64_t *pContL #else -int32_t taosAnalInit() { return 0; } -void taosAnalCleanup() {} +int32_t taosAnalyticsInit() { return 0; } +void taosAnalyticsCleanup() {} SJson *taosAnalSendReqRetJson(const char *url, EAnalHttpType type, SAnalBuf *pBuf) { return NULL; } int32_t taosAnalGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, int32_t urlLen) { return 0; } From 9adc756975c7cda6d08ea9f97062d66f4738118e Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 7 Nov 2024 17:37:33 +0800 Subject: [PATCH 18/44] enh: parameter validation --- source/libs/command/src/explain.c | 32 +++++++++++++++---------------- source/libs/function/src/tudf.c | 4 ++++ source/libs/function/src/udfd.c | 2 +- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 42c214fac7..3ab739334d 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -30,8 +30,8 @@ char *gJoinTypeStr[JOIN_TYPE_MAX_VALUE][JOIN_STYPE_MAX_VALUE] = { /*FULL*/ {"Full Join", "Full Join", NULL, NULL, NULL, NULL}, }; -int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes); -int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel); +static int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes); +static int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel); char *qExplainGetDynQryCtrlType(EDynQueryType type) { switch (type) { @@ -118,7 +118,7 @@ void qExplainFreeCtx(SExplainCtx *pCtx) { taosMemoryFree(pCtx); } -int32_t qExplainInitCtx(SExplainCtx **pCtx, SHashObj *groupHash, bool verbose, double ratio, EExplainMode mode) { +static int32_t qExplainInitCtx(SExplainCtx **pCtx, SHashObj *groupHash, bool verbose, double ratio, EExplainMode mode) { int32_t code = 0; SExplainCtx *ctx = taosMemoryCalloc(1, sizeof(SExplainCtx)); if (NULL == ctx) { @@ -158,7 +158,7 @@ _return: QRY_RET(code); } -int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNodeList **pChildren) { +static int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNodeList **pChildren) { int32_t tlen = 0; SNodeList *pPhysiChildren = pNode->pChildren; @@ -180,7 +180,7 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo return TSDB_CODE_SUCCESS; } -int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, SExplainGroup *group) { +static int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, SExplainGroup *group) { *pExecInfo = taosArrayInit(group->nodeNum, sizeof(SExplainExecInfo)); if (NULL == (*pExecInfo)) { qError("taosArrayInit %d explainExecInfo failed", group->nodeNum); @@ -217,7 +217,7 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S return TSDB_CODE_SUCCESS; } -int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pResNode) { +static int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pResNode) { if (NULL == pNode) { *pResNode = NULL; qError("physical node is NULL"); @@ -250,7 +250,7 @@ _return: QRY_RET(code); } -int32_t qExplainBufAppendExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { +static int32_t qExplainBufAppendExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { int32_t tlen = *len; int32_t nodeNum = taosArrayGetSize(pExecInfo); SExplainExecInfo maxExecInfo = {0}; @@ -275,7 +275,7 @@ int32_t qExplainBufAppendExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { return TSDB_CODE_SUCCESS; } -int32_t qExplainBufAppendVerboseExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { +static int32_t qExplainBufAppendVerboseExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { int32_t tlen = 0; bool gotVerbose = false; int32_t nodeNum = taosArrayGetSize(pExecInfo); @@ -297,7 +297,7 @@ int32_t qExplainBufAppendVerboseExecInfo(SArray *pExecInfo, char *tbuf, int32_t return TSDB_CODE_SUCCESS; } -int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t level) { +static int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t level) { SQueryExplainRowInfo row = {0}; row.buf = taosMemoryMalloc(len); if (NULL == row.buf) { @@ -362,7 +362,7 @@ static char* qExplainGetScanDataLoad(STableScanPhysiNode* pScan) { return "unknown"; } -int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { +static int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { int32_t tlen = 0; bool isVerboseLine = false; char *tbuf = ctx->tbuf; @@ -1900,7 +1900,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i return TSDB_CODE_SUCCESS; } -int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { +static int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { if (NULL == pResNode) { qError("explain res node is NULL"); QRY_ERR_RET(TSDB_CODE_APP_ERROR); @@ -1915,7 +1915,7 @@ int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32 return TSDB_CODE_SUCCESS; } -int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel) { +static int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel) { SExplainResNode *node = NULL; int32_t code = 0; SExplainCtx *ctx = (SExplainCtx *)pCtx; @@ -1940,7 +1940,7 @@ _return: QRY_RET(code); } -int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { +static int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { int32_t code = 0; SSDataBlock *pBlock = NULL; SExplainCtx *pCtx = (SExplainCtx *)ctx; @@ -1997,7 +1997,7 @@ _return: QRY_RET(code); } -int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { +static int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { int32_t code = 0; SNodeListNode *plans = NULL; int32_t taskNum = 0; @@ -2080,7 +2080,7 @@ _return: QRY_RET(code); } -int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) { +static int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) { if (EXPLAIN_MODE_ANALYZE != pCtx->mode) { return TSDB_CODE_SUCCESS; } @@ -2103,7 +2103,7 @@ int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) { return TSDB_CODE_SUCCESS; } -int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { +static int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { QRY_ERR_RET(qExplainAppendGroupResRows(pCtx, pCtx->rootGroupId, 0, false)); QRY_ERR_RET(qExplainAppendPlanRows(pCtx)); QRY_ERR_RET(qExplainGetRspFromCtx(pCtx, pRsp)); diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index a8198a804d..0e5b3ddbdb 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -64,6 +64,10 @@ static void udfWatchUdfd(void *args); void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int32_t termSignal) { fnInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal); SUdfdData *pData = process->data; + if(pData == NULL) { + fnError("udfd process data is NULL"); + return; + } if (exitStatus == 0 && termSignal == 0 || atomic_load_32(&pData->stopCalled)) { fnInfo("udfd process exit due to SIGINT or dnode-mgmt called stop"); } else { diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 6eef99e1f8..e3d533186d 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -1507,7 +1507,7 @@ static void removeListeningPipe() { int err = uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL); uv_fs_req_cleanup(&req); if(err) { - fnError("remove listening pipe %s failed, reason:%s, lino:%d", global.listenPipeName, uv_strerror(err), __LINE__); + fnInfo("remove listening pipe %s : %s, lino:%d", global.listenPipeName, uv_strerror(err), __LINE__); } } From 73881169c11837750f12509e7cf0201e21abdc10 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 7 Nov 2024 17:51:12 +0800 Subject: [PATCH 19/44] fix: install for taosanode --- packaging/tools/make_install.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 1b8fa2fb70..0874433e94 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -145,7 +145,14 @@ function kill_taosd() { function install_main_path() { #create install main dir and all sub dir - ${csudo}rm -rf ${install_main_dir} || : + ${csudo}rm -rf ${install_main_dir}/cfg || : + ${csudo}rm -rf ${install_main_dir}/bin || : + ${csudo}rm -rf ${install_main_dir}/driver || : + ${csudo}rm -rf ${install_main_dir}/examples || : + ${csudo}rm -rf ${install_main_dir}/include || : + ${csudo}rm -rf ${install_main_dir}/share || : + ${csudo}rm -rf ${install_main_dir}/log || : + ${csudo}mkdir -p ${install_main_dir} ${csudo}mkdir -p ${install_main_dir}/cfg ${csudo}mkdir -p ${install_main_dir}/bin From 6a601e0459e82b0e1fe499bc904071148587d492 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 17:54:01 +0800 Subject: [PATCH 20/44] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 7486b3b043..199715aba5 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -228,4 +228,30 @@ PAUSE STREAM [IF EXISTS] stream_name; RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; ``` -没有指定 IF EXISTS，如果该 stream 不存在，则报错。如果存在，则恢复流计算。指定了 IF EXISTS，如果 stream 不存在，则返回成功。如果存在，则恢复流计算。如果指定 IGNORE UNTREATED，则恢复流计算时，忽略流计算暂停期间写入的数据。 \ No newline at end of file +没有指定 IF EXISTS，如果该 stream 不存在，则报错。如果存在，则恢复流计算。指定了 IF EXISTS，如果 stream 不存在，则返回成功。如果存在，则恢复流计算。如果指定 IGNORE UNTREATED，则恢复流计算时，忽略流计算暂停期间写入的数据。 + +### 流计算升级故障恢复 + +升级导致流计算不兼容，需要删除流计算，然后重新创建流计算。步骤如下： +1.修改taos.cfg，添加disableStream 1 +2.启动taosd +3.启动taos， + +```sql +drop stream xxxx; ---- xxx指stream name +flush database stream_source_db; ----流计算读取数据的超级表所在的database； +flush database stream_dest_db; -----流计算写入数据的超级表所在的database； +``` + +举例： + +```sql +create stream streams1 into test1.streamst as select _wstart, count(a) c1 from test.st interval(1s) ; +drop database streams1; +flush database test; +flush database test1; +``` + +4.关闭taosd +5.修改taos.cfg，去掉disableStream 1，或将disableStream改为 0 +6.启动taosd \ No newline at end of file From 56be3e79d259c65c7b038d8435c03cc8a5414a10 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 17:56:10 +0800 Subject: [PATCH 21/44] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 199715aba5..7375619270 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -232,10 +232,10 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; ### 流计算升级故障恢复 -升级导致流计算不兼容，需要删除流计算，然后重新创建流计算。步骤如下： -1.修改taos.cfg，添加disableStream 1 -2.启动taosd -3.启动taos， +升级 TDengine 后，如果流计算不兼容，需要删除流计算，然后重新创建流计算。步骤如下： +1.修改 taos.cfg，添加disableStream 1 +2.启动 taosd +3.启动 taos， ```sql drop stream xxxx; ---- xxx指stream name @@ -252,6 +252,6 @@ flush database test; flush database test1; ``` -4.关闭taosd -5.修改taos.cfg，去掉disableStream 1，或将disableStream改为 0 -6.启动taosd \ No newline at end of file +4.关闭 taosd +5.修改 taos.cfg，去掉 disableStream 1，或将 disableStream 改为 0 +6.启动 taosd \ No newline at end of file From 7b6824dd9557416f3e57607b758d99491fe5552b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 17:57:26 +0800 Subject: [PATCH 22/44] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 7375619270..8b8ec3b7a4 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -233,8 +233,11 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; ### 流计算升级故障恢复 升级 TDengine 后，如果流计算不兼容，需要删除流计算，然后重新创建流计算。步骤如下： + 1.修改 taos.cfg，添加disableStream 1 + 2.启动 taosd + 3.启动 taos， ```sql @@ -253,5 +256,7 @@ flush database test1; ``` 4.关闭 taosd + 5.修改 taos.cfg，去掉 disableStream 1，或将 disableStream 改为 0 + 6.启动 taosd \ No newline at end of file From 875702106548136059f42696643103ba2060c33b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 17:58:56 +0800 Subject: [PATCH 23/44] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 8b8ec3b7a4..cc63a54008 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -236,7 +236,7 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; 1.修改 taos.cfg，添加disableStream 1 -2.启动 taosd +2.重启 taosd 3.启动 taos， From 334f82215e7bcff9035739ffeecdcdd828850350 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 7 Nov 2024 18:06:20 +0800 Subject: [PATCH 24/44] doc: minor changes --- docs/zh/06-advanced/03-stream.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index cc63a54008..671a6ec55c 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -234,16 +234,16 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; 升级 TDengine 后，如果流计算不兼容，需要删除流计算，然后重新创建流计算。步骤如下： -1.修改 taos.cfg，添加disableStream 1 +1.修改 taos.cfg，添加 disableStream 1 2.重启 taosd -3.启动 taos， +3.启动 taos ```sql -drop stream xxxx; ---- xxx指stream name -flush database stream_source_db; ----流计算读取数据的超级表所在的database； -flush database stream_dest_db; -----流计算写入数据的超级表所在的database； +drop stream xxxx; ---- xxx 指stream name +flush database stream_source_db; ---- 流计算读取数据的超级表所在的 database +flush database stream_dest_db; ---- 流计算写入数据的超级表所在的 database ``` 举例： From 9a8a91b619b07b9ec048dbc91738f079791912b3 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 18:10:11 +0800 Subject: [PATCH 25/44] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index cc63a54008..463fda3f47 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -236,7 +236,7 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; 1.修改 taos.cfg，添加disableStream 1 -2.重启 taosd +2.重启 taosd。如果启动失败，需要手动删除文件夹：$dataDir/vnode/vnode*/tq/stream，$dataDir指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似vnode1 、vnode2...vnode* 的目录 3.启动 taos， @@ -259,4 +259,4 @@ flush database test1; 5.修改 taos.cfg，去掉 disableStream 1，或将 disableStream 改为 0 -6.启动 taosd \ No newline at end of file +6.启动 taosd From 044645fa135bd169181fbe8042839b7c7880d7bf Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 18:12:17 +0800 Subject: [PATCH 26/44] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 463fda3f47..d1b2f31b2f 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -236,7 +236,7 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; 1.修改 taos.cfg，添加disableStream 1 -2.重启 taosd。如果启动失败，需要手动删除文件夹：$dataDir/vnode/vnode*/tq/stream，$dataDir指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似vnode1 、vnode2...vnode* 的目录 +2.重启 taosd。如果启动失败，需要手动删除文件夹：$dataDir/vnode/vnode*/tq/stream，$dataDir指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似vnode1 、vnode2...vnode* 的目录，全部需要删除里面的tq/stream目录 3.启动 taos， From ce0bd07de059c650b63ca0850089974d412ae0d4 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 18:15:39 +0800 Subject: [PATCH 27/44] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index d1b2f31b2f..c187adbe1a 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -236,7 +236,7 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; 1.修改 taos.cfg，添加disableStream 1 -2.重启 taosd。如果启动失败，需要手动删除文件夹：$dataDir/vnode/vnode*/tq/stream，$dataDir指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似vnode1 、vnode2...vnode* 的目录，全部需要删除里面的tq/stream目录 +2.重启 taosd。如果启动失败，需要手动修改文件夹名字：$dataDir/vnode/vnode*/tq/stream，$dataDir指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似vnode1 、vnode2...vnode* 的目录，全部需要修改里面的tq/stream目录的名字，改为vnode1.bk 、vnode2.bk ... vnode*.bk 3.启动 taos， From 1f7c9c93bd8e8870d138867f538daaaa9a05d35f Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 18:18:03 +0800 Subject: [PATCH 28/44] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 0ebb4b5b3e..758fbbd10c 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -259,4 +259,4 @@ flush database test1; 5.修改 taos.cfg，去掉 disableStream 1，或将 disableStream 改为 0 -6.启动 taosd +6.启动 taosd \ No newline at end of file From ba27958c67e97e3998257c59ef267ae4df38292c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 18:21:03 +0800 Subject: [PATCH 29/44] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 758fbbd10c..d22da5c4c8 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -236,7 +236,7 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; 1.修改 taos.cfg，添加 disableStream 1 -2.重启 taosd。如果启动失败，需要手动修改文件夹名字：$dataDir/vnode/vnode*/tq/stream，$dataDir指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似vnode1 、vnode2...vnode* 的目录，全部需要修改里面的tq/stream目录的名字，改为vnode1.bk 、vnode2.bk ... vnode*.bk +2.重启 taosd。如果启动失败，需要手动修改文件夹名字：$dataDir/vnode/vnode*/tq/stream，$dataDir 指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似 vnode1 、vnode2...vnode* 的目录，全部需要修改里面的 tq/stream 目录的名字，改为 vnode1.bk 、vnode2.bk ... vnode*.bk 3.启动 taos From 774ad21b22250a4b0877bd45bb06457f377acae5 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 19:03:13 +0800 Subject: [PATCH 30/44] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index d22da5c4c8..2f6e515fd7 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -236,7 +236,7 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; 1.修改 taos.cfg，添加 disableStream 1 -2.重启 taosd。如果启动失败，需要手动修改文件夹名字：$dataDir/vnode/vnode*/tq/stream，$dataDir 指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似 vnode1 、vnode2...vnode* 的目录，全部需要修改里面的 tq/stream 目录的名字，改为 vnode1.bk 、vnode2.bk ... vnode*.bk +2.重启 taosd。如果启动失败，修改 stream 目录的名称，避免 taosd 启动的时候尝试加载 stream 目录下的流计算数据信息。不使用删除操作避免误操作导致的风险。需要修改的文件夹：$dataDir/vnode/vnode*/tq/stream，$dataDir 指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似 vnode1 、vnode2...vnode* 的目录，全部需要修改里面的 tq/stream 目录的名字，改为 stream.bk 3.启动 taos From e64ad3a27d40df6d9bddbdfdece17a9724e434bc Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 7 Nov 2024 19:04:23 +0800 Subject: [PATCH 31/44] doc(stream):recover stream --- docs/zh/06-advanced/03-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index 2f6e515fd7..c47831dde3 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -236,7 +236,7 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; 1.修改 taos.cfg，添加 disableStream 1 -2.重启 taosd。如果启动失败，修改 stream 目录的名称，避免 taosd 启动的时候尝试加载 stream 目录下的流计算数据信息。不使用删除操作避免误操作导致的风险。需要修改的文件夹：$dataDir/vnode/vnode*/tq/stream，$dataDir 指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似 vnode1 、vnode2...vnode* 的目录，全部需要修改里面的 tq/stream 目录的名字，改为 stream.bk +2.重启 taosd。如果启动失败，修改 stream 目录的名称，避免 taosd 启动的时候尝试加载 stream 目录下的流计算数据信息。不使用删除操作避免误操作导致的风险。需要修改的文件夹：$dataDir/vnode/vnode*/tq/stream，$dataDir 指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似 vnode1 、vnode2...vnode* 的目录，全部需要修改里面的 tq/stream 目录的名字，改为 tq/stream.bk 3.启动 taos From 9c5c9f62addf9f8ac4e5ac80e2254d4894ba0f43 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 7 Nov 2024 19:48:32 +0800 Subject: [PATCH 32/44] feat: kill rsync by pid. --- include/os/osSemaphore.h | 1 + source/common/src/rsync.c | 15 +++++++-- source/os/src/osSemaphore.c | 51 +++++++++++++++++++++++++++++ source/os/test/osSemaphoreTests.cpp | 7 ++++ 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/include/os/osSemaphore.h b/include/os/osSemaphore.h index d893f42740..48836eeb0c 100644 --- a/include/os/osSemaphore.h +++ b/include/os/osSemaphore.h @@ -104,6 +104,7 @@ int64_t taosGetPthreadId(TdThread thread); void taosResetPthread(TdThread *thread); bool taosComparePthread(TdThread first, TdThread second); int32_t taosGetPId(); +int32_t taosGetPIdByName(const char* name, int32_t* pPId); int32_t taosGetAppName(char *name, int32_t *len); #ifdef __cplusplus diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c index b7352acf25..ecf7fc21a6 100644 --- a/source/common/src/rsync.c +++ b/source/common/src/rsync.c @@ -119,11 +119,20 @@ static int32_t execCommand(char* command) { } void stopRsync() { - int32_t code = + int32_t pid = 0; + int32_t code = 0; + char buf[128] = {0}; + #ifdef WINDOWS - system("taskkill /f /im rsync.exe"); + code = system("taskkill /f /im rsync.exe"); #else - system("pkill rsync"); + code = taosGetPIdByName("rsync", &pid); + if (code == 0) { + int32_t ret = tsnprintf(buf, tListLen(buf), "kill -9 %d", pid); + if (ret > 0) { + system(buf); + } + } #endif if (code != 0) { diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 538d5bf63e..ff3d7e2976 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -72,6 +72,8 @@ int32_t taosGetAppName(char* name, int32_t* len) { return 0; } +int32_t taosGetPIdByName(const char* name, int32_t* pPId) { return -1;} + int32_t tsem_wait(tsem_t* sem) { DWORD ret = WaitForSingleObject(*sem, INFINITE); if (ret == WAIT_OBJECT_0) { @@ -173,6 +175,8 @@ int32_t taosGetAppName(char *name, int32_t *len) { return 0; } +int32_t taosGetPIdByName(const char* name, int32_t* pPId) {return -1;} + #else /* @@ -228,6 +232,53 @@ int32_t taosGetAppName(char* name, int32_t* len) { return 0; } +int32_t taosGetPIdByName(const char* name, int32_t* pPId) { + DIR* dir = NULL; + struct dirent* ptr = NULL; + FILE* fp = NULL; + char filepath[512]; + char bufx[50]; + char buf[1024] = {0}; + + *pPId = -1; + dir = opendir("/proc"); + if (dir == NULL) { + return TAOS_SYSTEM_ERROR(errno); + } + + while ((ptr = readdir(dir)) != NULL) { + if ((strcmp(ptr->d_name, ".") == 0) || (strcmp(ptr->d_name, "..") == 0)) { + continue; + } + + if (DT_DIR != ptr->d_type) { + continue; + } + + int32_t ret = tsnprintf(filepath, tListLen(filepath), "/proc/%s/status", ptr->d_name); + if (ret == -1) { + continue; + } + + fp = fopen(filepath, "r"); + if (NULL != fp) { + if (fgets(buf, tListLen(buf) - 1, fp) == NULL) { + fclose(fp); + continue; + } + + sscanf(buf, "%*s %s", bufx); + if (!strcmp(bufx, name)) { + char* end = NULL; + *pPId = taosStr2Int32(ptr->d_name, &end, 10); + } + fclose(fp); + } + } + + return 0; +} + int32_t tsem_init(tsem_t* psem, int flags, unsigned int count) { if (sem_init(psem, flags, count) == 0) { return 0; diff --git a/source/os/test/osSemaphoreTests.cpp b/source/os/test/osSemaphoreTests.cpp index 1576d2845d..b4b07f8572 100644 --- a/source/os/test/osSemaphoreTests.cpp +++ b/source/os/test/osSemaphoreTests.cpp @@ -245,3 +245,10 @@ TEST(osSemaphoreTests, Performance4_2) { (void)tsem2_destroy(&sem); } } + +TEST(osSemaphoreTests, GetPID) { + pid_t pid = 0; + int32_t ret = taosGetPIdByName("osSemaphoreTest", &pid); + EXPECT_EQ(ret, 0); + EXPECT_EQ(pid, taosGetPId()); +} From c312896d73f28c00ff2d86ef4371dfe2a1d2391c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 8 Nov 2024 10:31:48 +0800 Subject: [PATCH 33/44] feat: set correct rsp code. --- source/common/src/rsync.c | 1 + source/os/src/osSemaphore.c | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c index ecf7fc21a6..15c296d730 100644 --- a/source/common/src/rsync.c +++ b/source/common/src/rsync.c @@ -130,6 +130,7 @@ void stopRsync() { if (code == 0) { int32_t ret = tsnprintf(buf, tListLen(buf), "kill -9 %d", pid); if (ret > 0) { + uInfo("kill rsync program pid:%d", pid); system(buf); } } diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index ff3d7e2976..58c832af89 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -276,7 +276,11 @@ int32_t taosGetPIdByName(const char* name, int32_t* pPId) { } } - return 0; + if ((*pPId) == -1) { + return TAOS_SYSTEM_ERROR(ESRCH); + } else { + return TSDB_CODE_SUCCESS; + } } int32_t tsem_init(tsem_t* psem, int flags, unsigned int count) { From d8098ab3e145bb54b20ac72ea53d16b565d46dff Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 8 Nov 2024 12:48:00 +0800 Subject: [PATCH 34/44] fix(util): check return value. --- source/os/src/osSemaphore.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 58c832af89..1b9e8d3273 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -263,7 +263,7 @@ int32_t taosGetPIdByName(const char* name, int32_t* pPId) { fp = fopen(filepath, "r"); if (NULL != fp) { if (fgets(buf, tListLen(buf) - 1, fp) == NULL) { - fclose(fp); + TAOS_UNUSED(fclose(fp)); continue; } @@ -272,7 +272,7 @@ int32_t taosGetPIdByName(const char* name, int32_t* pPId) { char* end = NULL; *pPId = taosStr2Int32(ptr->d_name, &end, 10); } - fclose(fp); + TAOS_UNUSED(fclose(fp)); } } From 3559188dfa007c2a1679d0d12fcad630b61b9d58 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 8 Nov 2024 14:23:37 +0800 Subject: [PATCH 35/44] fix(stream): check return value. --- source/common/src/rsync.c | 2 +- source/os/src/osSemaphore.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c index 15c296d730..8b453f7d82 100644 --- a/source/common/src/rsync.c +++ b/source/common/src/rsync.c @@ -131,7 +131,7 @@ void stopRsync() { int32_t ret = tsnprintf(buf, tListLen(buf), "kill -9 %d", pid); if (ret > 0) { uInfo("kill rsync program pid:%d", pid); - system(buf); + code = system(buf); } } #endif diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 1b9e8d3273..e37182ce16 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -267,7 +267,7 @@ int32_t taosGetPIdByName(const char* name, int32_t* pPId) { continue; } - sscanf(buf, "%*s %s", bufx); + ret = sscanf(buf, "%*s %s", bufx); if (!strcmp(bufx, name)) { char* end = NULL; *pPId = taosStr2Int32(ptr->d_name, &end, 10); From 021024e6dc2b73a65f1014e0310ff72c170b64d5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 8 Nov 2024 17:17:47 +0800 Subject: [PATCH 36/44] add ip white list read --- docs/zh/08-operation/16-security.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/zh/08-operation/16-security.md b/docs/zh/08-operation/16-security.md index 4f47a644f7..1c8ad2200f 100644 --- a/docs/zh/08-operation/16-security.md +++ b/docs/zh/08-operation/16-security.md @@ -26,6 +26,22 @@ SHOW USERS; ```sql ALTER USER TEST DROP HOST HOST_NAME1 ``` +说明 +1. 开源版和企业版本都能添加成功，且可以查询到，但是开源版本不会对IP做任何限制。 +2. create user u_write pass 'taosdata1' host 'iprange1','iprange2', 可以一次添加多个iprange, 服务端会做去重，去重的逻辑是需要iprange 完全一样 +3. 默认会把127.0.0.1 添加到白名单列表，且在白名单列表可以查询 +4. 集群的节点IP集合会自动添加到白名单列表，但是查询不到。 +5. taosadaper 和 taosd 不在一个机器的时候，需要把taosadaper IP手动添加到taosd 白名单列表中 +6. 集群情况下，各个节点 enableWhiteList 成一样，或者全为false,或者全为true, 要不然集群无法启动 +7. 白名单变更生效时间1s，不超过2s, 每次变更对收发性能有些微影响（多一次判断，可以忽略），变更完之后、影响忽略不计, 变更过程中对集群没有影响，对正在访问客户端也没有影响（假设这些客户端的IP包含在white list内） +8. 如果添加两个ip range, 192.168.1.1/16(假设为A), 192.168.1.1/24(假设为B), 严格来说，A包含了B，但是考虑情况太复杂，并不会对A和B做合并 +9. 要删除的时候，必须严格匹配。 也就是如果添加的是192.168.1.1/24, 要删除也是192.168.1.1/24 +10. 只有root 才有权限对其他用户增删ip white list +11. 兼容之前的版本，但是不支持从当前版本回退到之前版本 +12. x.x.x.x/32 和x.x.x.x 属于同一个iprange, 显示为x.x.x.x +13. 如果客户端拿到的 0.0.0.0/0, 说明没有开启白名单。 +14. 如果白名单发生了改变， 客户端会在heartbeat里检测到。 +15. 针对一个user, 添加的IP个数上限是2048 ## 审计日志 From 5d0f3206d5af86e62571310c17a1e3ed446c5f13 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 9 Nov 2024 11:27:04 +0800 Subject: [PATCH 37/44] doc: minor changes --- docs/zh/08-operation/16-security.md | 30 ++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/zh/08-operation/16-security.md b/docs/zh/08-operation/16-security.md index 1c8ad2200f..e3cd72d9dc 100644 --- a/docs/zh/08-operation/16-security.md +++ b/docs/zh/08-operation/16-security.md @@ -27,21 +27,21 @@ SHOW USERS; ALTER USER TEST DROP HOST HOST_NAME1 ``` 说明 -1. 开源版和企业版本都能添加成功，且可以查询到，但是开源版本不会对IP做任何限制。 -2. create user u_write pass 'taosdata1' host 'iprange1','iprange2', 可以一次添加多个iprange, 服务端会做去重，去重的逻辑是需要iprange 完全一样 -3. 默认会把127.0.0.1 添加到白名单列表，且在白名单列表可以查询 -4. 集群的节点IP集合会自动添加到白名单列表，但是查询不到。 -5. taosadaper 和 taosd 不在一个机器的时候，需要把taosadaper IP手动添加到taosd 白名单列表中 -6. 集群情况下，各个节点 enableWhiteList 成一样，或者全为false,或者全为true, 要不然集群无法启动 -7. 白名单变更生效时间1s，不超过2s, 每次变更对收发性能有些微影响（多一次判断，可以忽略），变更完之后、影响忽略不计, 变更过程中对集群没有影响，对正在访问客户端也没有影响（假设这些客户端的IP包含在white list内） -8. 如果添加两个ip range, 192.168.1.1/16(假设为A), 192.168.1.1/24(假设为B), 严格来说，A包含了B，但是考虑情况太复杂，并不会对A和B做合并 -9. 要删除的时候，必须严格匹配。 也就是如果添加的是192.168.1.1/24, 要删除也是192.168.1.1/24 -10. 只有root 才有权限对其他用户增删ip white list -11. 兼容之前的版本，但是不支持从当前版本回退到之前版本 -12. x.x.x.x/32 和x.x.x.x 属于同一个iprange, 显示为x.x.x.x -13. 如果客户端拿到的 0.0.0.0/0, 说明没有开启白名单。 -14. 如果白名单发生了改变， 客户端会在heartbeat里检测到。 -15. 针对一个user, 添加的IP个数上限是2048 +- 开源版和企业版本都能添加成功，且可以查询到，但是开源版本不会对 IP 做任何限制。 +- create user u_write pass 'taosdata1' host 'iprange1','iprange2', 可以一次添加多个 iprange, 服务端会做去重，去重的逻辑是需要 iprange 完全一样 +- 默认会把 127.0.0.1 添加到白名单列表，且在白名单列表可以查询 +- 集群的节点 IP 集合会自动添加到白名单列表，但是查询不到。 +- taosadaper 和 taosd 不在一个机器的时候，需要把 taosadaper IP 手动添加到 taosd 白名单列表中 +- 集群情况下，各个节点 enableWhiteList 成一样，或者全为 false,或者全为 true, 要不然集群无法启动 +- 白名单变更生效时间 1s，不超过 2s, 每次变更对收发性能有些微影响（多一次判断，可以忽略），变更完之后、影响忽略不计, 变更过程中对集群没有影响，对正在访问客户端也没有影响（假设这些客户端的 IP 包含在 white list 内） +- 如果添加两个 ip range, 192.168.1.1/16(假设为 A), 192.168.1.1/24(假设为 B), 严格来说，A 包含了 B，但是考虑情况太复杂，并不会对 A 和 B 做合并 +- 要删除的时候，必须严格匹配。 也就是如果添加的是 192.168.1.1/24, 要删除也是 192.168.1.1/24 +- 只有 root 才有权限对其他用户增删 ip white list +- 兼容之前的版本，但是不支持从当前版本回退到之前版本 +- x.x.x.x/32 和 x.x.x.x 属于同一个 iprange, 显示为 x.x.x.x +- 如果客户端拿到的 0.0.0.0/0, 说明没有开启白名单 +- 如果白名单发生了改变， 客户端会在 heartbeat 里检测到。 +- 针对一个 user, 添加的 IP 个数上限是 2048 ## 审计日志 From 9ff749be73ea57811473544766e3b3eefac5d057 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 10 Nov 2024 10:12:21 +0800 Subject: [PATCH 38/44] fix:disable unit test for windows/mac --- source/os/test/osSemaphoreTests.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/os/test/osSemaphoreTests.cpp b/source/os/test/osSemaphoreTests.cpp index b4b07f8572..1e412f5149 100644 --- a/source/os/test/osSemaphoreTests.cpp +++ b/source/os/test/osSemaphoreTests.cpp @@ -247,8 +247,10 @@ TEST(osSemaphoreTests, Performance4_2) { } TEST(osSemaphoreTests, GetPID) { +#ifdef LINUX pid_t pid = 0; int32_t ret = taosGetPIdByName("osSemaphoreTest", &pid); EXPECT_EQ(ret, 0); EXPECT_EQ(pid, taosGetPId()); +#endif } From 4b50edccf0ddc601c32a0da43453af17ee068ac6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 10 Nov 2024 10:49:31 +0800 Subject: [PATCH 39/44] fix: fix memory leak. --- source/os/src/osSemaphore.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index e37182ce16..708c18aa3f 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -276,6 +276,8 @@ int32_t taosGetPIdByName(const char* name, int32_t* pPId) { } } + closedir(dir); + if ((*pPId) == -1) { return TAOS_SYSTEM_ERROR(ESRCH); } else { From 006900408a36279ae4a054f000444d56022426e7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 10 Nov 2024 12:23:01 +0800 Subject: [PATCH 40/44] fix: check return value. --- source/os/src/osSemaphore.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 708c18aa3f..e61fd627eb 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -276,7 +276,7 @@ int32_t taosGetPIdByName(const char* name, int32_t* pPId) { } } - closedir(dir); + TAOS_UNUSED(closedir(dir)); if ((*pPId) == -1) { return TAOS_SYSTEM_ERROR(ESRCH); From 1a433666eb55c32a373d8db12c24265e0374d88c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 10 Nov 2024 16:50:36 +0800 Subject: [PATCH 41/44] opt get conn rule --- source/libs/transport/src/transCli.c | 122 +++++++++++++++------------ 1 file changed, 66 insertions(+), 56 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 2aeffc6395..560a501f2b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -254,6 +254,7 @@ static FORCE_INLINE void cliMayUpdateFqdnCache(SHashObj* cache, char* dst); // process data read from server, add decompress etc later // handle except about conn +#define REQS_ON_CONN(conn) (conn ? (transQueueSize(&conn->reqsToSend) + transQueueSize(&conn->reqsSentOut)) : 0) static void doNotifyCb(SCliReq* pReq, SCliThrd* pThrd, int32_t code); // handle req from app static void cliHandleReq(SCliThrd* pThrd, SCliReq* pReq); @@ -289,7 +290,7 @@ int32_t cliMayGetStateByQid(SCliThrd* pThrd, SCliReq* pReq, SCliConn** pConn); static SCliConn* getConnFromHeapCache(SHashObj* pConnHeapCache, char* key); static int32_t addConnToHeapCache(SHashObj* pConnHeapCacahe, SCliConn* pConn); static int32_t delConnFromHeapCache(SHashObj* pConnHeapCache, SCliConn* pConn); -static int32_t balanceConnHeapCache(SHashObj* pConnHeapCache, SCliConn* pConn); +static int8_t balanceConnHeapCache(SHashObj* pConnHeapCache, SCliConn* pConn, SCliConn** pNewConn); // thread obj static int32_t createThrdObj(void* trans, SCliThrd** pThrd); @@ -327,14 +328,18 @@ typedef struct { int64_t lastConnFailTs; } SHeap; -int32_t compareHeapNode(const HeapNode* a, const HeapNode* b); -int32_t transHeapInit(SHeap* heap, int32_t (*cmpFunc)(const HeapNode* a, const HeapNode* b)); -void transHeapDestroy(SHeap* heap); -int32_t transHeapGet(SHeap* heap, SCliConn** p); -int32_t transHeapInsert(SHeap* heap, SCliConn* p); -int32_t transHeapDelete(SHeap* heap, SCliConn* p); -int32_t transHeapBalance(SHeap* heap, SCliConn* p); -int32_t transHeapUpdateFailTs(SHeap* heap, SCliConn* p); +static int32_t compareHeapNode(const HeapNode* a, const HeapNode* b); +static int32_t transHeapInit(SHeap* heap, int32_t (*cmpFunc)(const HeapNode* a, const HeapNode* b)); +static void transHeapDestroy(SHeap* heap); + +static int32_t transHeapGet(SHeap* heap, SCliConn** p); +static int32_t transHeapInsert(SHeap* heap, SCliConn* p); +static int32_t transHeapDelete(SHeap* heap, SCliConn* p); +static int32_t transHeapBalance(SHeap* heap, SCliConn* p); +static int32_t transHeapUpdateFailTs(SHeap* heap, SCliConn* p); +static int32_t transHeapMayBalance(SHeap* heap, SCliConn* p); + +static FORCE_INLINE void logConnMissHit(SCliConn* pConn); #define CLI_RELEASE_UV(loop) \ do { \ @@ -498,6 +503,8 @@ int8_t cliMayRecycleConn(SCliConn* conn) { if (code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) { tDebug("%s conn %p failed to remove conn from heap cache since %s", CONN_GET_INST_LABEL(conn), conn, tstrerror(code)); + + TAOS_UNUSED(transHeapMayBalance(conn->heap, conn)); return 1; } else { if (code != 0) { @@ -510,31 +517,10 @@ int8_t cliMayRecycleConn(SCliConn* conn) { } else if ((transQueueSize(&conn->reqsToSend) == 0) && (transQueueSize(&conn->reqsSentOut) == 0) && (taosHashGetSize(conn->pQTable) != 0)) { tDebug("%s conn %p do balance directly", CONN_GET_INST_LABEL(conn), conn); - TAOS_UNUSED(transHeapBalance(conn->heap, conn)); + TAOS_UNUSED(transHeapMayBalance(conn->heap, conn)); } else { - SCliConn* topConn = NULL; - if (conn->heap != NULL) { - code = transHeapGet(conn->heap, &topConn); - if (code != 0) { - tDebug("%s conn %p failed to get top conn since %s", CONN_GET_INST_LABEL(conn), conn, tstrerror(code)); - return 0; - } - - if (topConn == conn) { - return 0; - } - int32_t topReqs = transQueueSize(&topConn->reqsSentOut) + transQueueSize(&topConn->reqsToSend); - int32_t currReqs = transQueueSize(&conn->reqsSentOut) + transQueueSize(&conn->reqsToSend); - if (topReqs <= currReqs) { - tTrace("%s conn %p not balance conn heap since top conn has less req, topConnReqs:%d, currConnReqs:%d", - CONN_GET_INST_LABEL(conn), conn, topReqs, currReqs); - return 0; - } else { - tDebug("%s conn %p do balance conn heap since top conn has more reqs, topConnReqs:%d, currConnReqs:%d", - CONN_GET_INST_LABEL(conn), conn, topReqs, currReqs); - TAOS_UNUSED(transHeapBalance(conn->heap, conn)); - } - } + tTrace("%s conn %p may do balance", CONN_GET_INST_LABEL(conn), conn); + TAOS_UNUSED(transHeapMayBalance(conn->heap, conn)); } return 0; } @@ -785,15 +771,8 @@ void cliConnCheckTimoutMsg(SCliConn* conn) { if (transQueueSize(&conn->reqsSentOut) == 0) { return; } - code = cliConnRemoveTimeoutMsg(conn); - if (code != 0) { - tDebug("%s conn %p do remove timeout msg", CONN_GET_INST_LABEL(conn), conn); - if (!cliMayRecycleConn(conn)) { - TAOS_UNUSED(transHeapBalance(conn->heap, conn)); - } - } else { - TAOS_UNUSED(cliMayRecycleConn(conn)); - } + TAOS_UNUSED(cliConnRemoveTimeoutMsg(conn)); + TAOS_UNUSED(cliMayRecycleConn(conn)); } void cliConnTimeout__checkReq(uv_timer_t* handle) { SCliConn* conn = handle->data; @@ -3804,6 +3783,8 @@ static FORCE_INLINE int8_t shouldSWitchToOtherConn(SCliConn* pConn, char* key) { int32_t totalReqs = reqsNum + reqsSentOut; if (totalReqs >= pInst->shareConnLimit) { + logConnMissHit(pConn); + if (pConn->list == NULL && pConn->dstAddr != NULL) { pConn->list = taosHashGet((SHashObj*)pThrd->pool, pConn->dstAddr, strlen(pConn->dstAddr)); if (pConn->list != NULL) { @@ -3860,11 +3841,12 @@ static SCliConn* getConnFromHeapCache(SHashObj* pConnHeapCache, char* key) { } else { tTrace("conn %p get conn from heap cache for key:%s", pConn, key); if (shouldSWitchToOtherConn(pConn, key)) { - code = balanceConnHeapCache(pConnHeapCache, pConn); - if (code != 0) { - tTrace("failed to balance conn heap cache for key:%s", key); + SCliConn* pNewConn = NULL; + code = balanceConnHeapCache(pConnHeapCache, pConn, &pNewConn); + if (code == 1) { + tTrace("conn %p start to handle reqs", key, pNewConn); + return pNewConn; } - logConnMissHit(pConn); return NULL; } } @@ -3916,15 +3898,19 @@ static int32_t delConnFromHeapCache(SHashObj* pConnHeapCache, SCliConn* pConn) { return code; } -static int32_t balanceConnHeapCache(SHashObj* pConnHeapCache, SCliConn* pConn) { +static int8_t balanceConnHeapCache(SHashObj* pConnHeapCache, SCliConn* pConn, SCliConn** pNewConn) { + SCliThrd* pThrd = pConn->hostThrd; + STrans* pInst = pThrd->pInst; + SCliConn* pTopConn = NULL; if (pConn->heap != NULL && pConn->inHeap != 0) { - SHeap* heap = pConn->heap; - tTrace("conn %p'heap may should do balance, numOfConn:%d", pConn, (int)(heap->heap->nelts)); - int64_t now = taosGetTimestampMs(); - if (((now - heap->lastUpdateTs) / 1000) > 30) { - heap->lastUpdateTs = now; - tTrace("conn %p'heap do balance, numOfConn:%d", pConn, (int)(heap->heap->nelts)); - return transHeapBalance(pConn->heap, pConn); + TAOS_UNUSED(transHeapBalance(pConn->heap, pConn)); + if (transHeapGet(pConn->heap, &pTopConn) == 0 && pConn != pTopConn) { + int32_t curReqs = REQS_ON_CONN(pConn); + int32_t topReqs = REQS_ON_CONN(pTopConn); + if (curReqs > topReqs && topReqs < pInst->shareConnLimit) { + *pNewConn = pTopConn; + return 1; + } } } return 0; @@ -3934,8 +3920,8 @@ int32_t compareHeapNode(const HeapNode* a, const HeapNode* b) { SCliConn* args1 = container_of(a, SCliConn, node); SCliConn* args2 = container_of(b, SCliConn, node); - int32_t totalReq1 = transQueueSize(&args1->reqsToSend) + transQueueSize(&args1->reqsSentOut); - int32_t totalReq2 = transQueueSize(&args2->reqsToSend) + transQueueSize(&args2->reqsSentOut); + int32_t totalReq1 = REQS_ON_CONN(args1); + int32_t totalReq2 = REQS_ON_CONN(args2); if (totalReq1 > totalReq2) { return 0; } @@ -4016,6 +4002,30 @@ int32_t transHeapUpdateFailTs(SHeap* heap, SCliConn* p) { heap->lastConnFailTs = taosGetTimestampMs(); return 0; } +int32_t transHeapMayBalance(SHeap* heap, SCliConn* p) { + if (p->inHeap == 0 || heap == NULL || heap->heap == NULL) { + return 0; + } + SCliThrd* pThrd = p->hostThrd; + STrans* pInst = pThrd->pInst; + int32_t balanceLimit = pInst->shareConnLimit >= 4 ? pInst->shareConnLimit / 2 : 2; + + SCliConn* topConn = NULL; + int32_t code = transHeapGet(heap, &topConn); + if (code != 0) { + return code; + } + + if (topConn == p) return code; + + int32_t reqsOnTop = REQS_ON_CONN(topConn); + int32_t reqsOnCur = REQS_ON_CONN(p); + + if (reqsOnTop >= balanceLimit && reqsOnCur < balanceLimit) { + TAOS_UNUSED(transHeapBalance(heap, p)); + } + return code; +} int32_t transHeapBalance(SHeap* heap, SCliConn* p) { if (p->inHeap == 0 || heap == NULL || heap->heap == NULL) { From ca860881a7cb970c5a15b342e0471d4442b8c074 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 10 Nov 2024 16:53:01 +0800 Subject: [PATCH 42/44] fix compile error --- source/libs/transport/src/transCli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 560a501f2b..057154647f 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -3844,7 +3844,7 @@ static SCliConn* getConnFromHeapCache(SHashObj* pConnHeapCache, char* key) { SCliConn* pNewConn = NULL; code = balanceConnHeapCache(pConnHeapCache, pConn, &pNewConn); if (code == 1) { - tTrace("conn %p start to handle reqs", key, pNewConn); + tTrace("conn %p start to handle reqs", pNewConn); return pNewConn; } return NULL; From abb79a4402fb8a539d50af0dde0e1d6e65216817 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 10 Nov 2024 21:41:51 +0800 Subject: [PATCH 43/44] opt get conn rule --- source/libs/transport/src/transCli.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 057154647f..0038dc55f1 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -499,6 +499,7 @@ int8_t cliMayRecycleConn(SCliConn* conn) { if (transQueueSize(&conn->reqsToSend) == 0 && transQueueSize(&conn->reqsSentOut) == 0 && taosHashGetSize(conn->pQTable) == 0) { cliResetConnTimer(conn); + conn->forceDelFromHeap = 1; code = delConnFromHeapCache(pThrd->connHeapCache, conn); if (code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) { tDebug("%s conn %p failed to remove conn from heap cache since %s", CONN_GET_INST_LABEL(conn), conn, From 0087ba3f62cee71b63f472caa89fbce79952271d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 10 Nov 2024 21:44:35 +0800 Subject: [PATCH 44/44] opt get conn rule --- source/libs/transport/src/transCli.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 0038dc55f1..c03d3418fa 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -511,6 +511,7 @@ int8_t cliMayRecycleConn(SCliConn* conn) { if (code != 0) { tDebug("%s conn %p failed to remove conn from heap cache since %s", CONN_GET_INST_LABEL(conn), conn, tstrerror(code)); + return 0; } } addConnToPool(pThrd->pool, conn);