From 4a54ce26d5d842be7ce629ea0e39f87a35a0d965 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 18 Jul 2022 14:25:00 +0800 Subject: [PATCH 01/47] fix: fix sys table show stables issue --- source/libs/executor/src/scanoperator.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b80b9af237..f51dd076a7 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2126,6 +2126,21 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { } } + +static SSDataBlock* sysTableScanUserSTables(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + pInfo->pRes->info.rows = 0; + pOperator->status == OP_EXEC_DONE; + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { // build message and send to mnode to fetch the content of system tables. SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2136,6 +2151,8 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { return sysTableScanUserTables(pOperator); } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { return sysTableScanUserTags(pOperator); + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, TSDB_TABLE_FNAME_LEN) == 0) { + return sysTableScanUserSTables(pOperator); } else { // load the meta from mnode of the given epset if (pOperator->status == OP_EXEC_DONE) { return NULL; From 144405443793d90f4795f2844c30345c8e3a498a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 18 Jul 2022 18:57:47 +0800 Subject: [PATCH 02/47] fix: fix stmt memory leak --- source/client/src/clientStmt.c | 13 +++++++++---- source/libs/nodes/src/nodesUtilFuncs.c | 8 ++------ source/libs/parser/src/parInsert.c | 4 +++- source/libs/parser/src/parser.c | 6 ++++++ tests/script/api/batchprepare.c | 9 ++++----- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index bf00965c7a..7a83006961 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -6,11 +6,16 @@ #include "clientStmt.h" static int32_t stmtCreateRequest(STscStmt* pStmt) { + int32_t code = 0; + if (pStmt->exec.pRequest == NULL) { - return buildRequest(pStmt->taos->id, pStmt->sql.sqlStr, pStmt->sql.sqlLen, NULL, false, &pStmt->exec.pRequest); - } else { - return TSDB_CODE_SUCCESS; + code = buildRequest(pStmt->taos->id, pStmt->sql.sqlStr, pStmt->sql.sqlLen, NULL, false, &pStmt->exec.pRequest); + if (TSDB_CODE_SUCCESS == code) { + pStmt->exec.pRequest->syncQuery = true; + } } + + return code; } int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { @@ -227,7 +232,7 @@ int32_t stmtParseSql(STscStmt* pStmt) { }; STMT_ERR_RET(stmtCreateRequest(pStmt)); - + STMT_ERR_RET(parseSql(pStmt->exec.pRequest, false, &pStmt->sql.pQuery, &stmtCb)); pStmt->bInfo.needParse = false; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 1dc3db033b..c96dc194ca 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -388,11 +388,6 @@ static void destroyDataSinkNode(SDataSinkNode* pNode) { nodesDestroyNode((SNode* static void destroyExprNode(SExprNode* pExpr) { taosArrayDestroy(pExpr->pAssociation); } -static void nodesDestroyNodePointer(void* node) { - SNode* pNode = *(SNode**)node; - nodesDestroyNode(pNode); -} - void nodesDestroyNode(SNode* pNode) { if (NULL == pNode) { return; @@ -716,6 +711,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_QUERY: { SQuery* pQuery = (SQuery*)pNode; nodesDestroyNode(pQuery->pRoot); + nodesDestroyNode(pQuery->pPrepareRoot); taosMemoryFreeClear(pQuery->pResSchema); if (NULL != pQuery->pCmdMsg) { taosMemoryFreeClear(pQuery->pCmdMsg->pMsg); @@ -723,7 +719,7 @@ void nodesDestroyNode(SNode* pNode) { } taosArrayDestroy(pQuery->pDbList); taosArrayDestroy(pQuery->pTableList); - taosArrayDestroyEx(pQuery->pPlaceholderValues, nodesDestroyNodePointer); + taosArrayDestroy(pQuery->pPlaceholderValues); break; } case QUERY_NODE_LOGIC_PLAN_SCAN: { diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 5ac4476d14..3defd0224b 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1497,7 +1497,6 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { memset(&pCxt->tags, 0, sizeof(pCxt->tags)); pCxt->pVgroupsHashObj = NULL; pCxt->pTableBlockHashObj = NULL; - pCxt->pTableMeta = NULL; return TSDB_CODE_SUCCESS; } @@ -1554,7 +1553,10 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache if (NULL == *pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } + } else { + nodesDestroyNode((*pQuery)->pRoot); } + (*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE; (*pQuery)->haveResultSet = false; (*pQuery)->msgType = TDMT_VND_SUBMIT; diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index fdba0e2fcc..78d1e83436 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -82,11 +82,16 @@ static int32_t parseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, SParseMetaCa } static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { + if (IS_VAR_DATA_TYPE(pVal->node.resType.type)) { + taosMemoryFreeClear(pVal->datum.p); + } + if (pParam->is_null && 1 == *(pParam->is_null)) { pVal->node.resType.type = TSDB_DATA_TYPE_NULL; pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_NULL].bytes; return TSDB_CODE_SUCCESS; } + int32_t inputSize = (NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes); pVal->node.resType.type = pParam->buffer_type; pVal->node.resType.bytes = inputSize; @@ -239,6 +244,7 @@ int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx } if (TSDB_CODE_SUCCESS == code && (colIdx < 0 || colIdx + 1 == pQuery->placeholderNum)) { + nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = nodesCloneNode(pQuery->pPrepareRoot); if (NULL == pQuery->pRoot) { code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 29c1fdb015..a330b6416e 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -218,7 +218,7 @@ typedef struct { } CaseCtrl; #if 0 -CaseCtrl gCaseCtrl = { // default +CaseCtrl gCaseCtrl = { .precision = TIME_PRECISION_MICRO, .bindNullNum = 0, .printCreateTblSql = false, @@ -251,7 +251,7 @@ CaseCtrl gCaseCtrl = { // default #if 1 -CaseCtrl gCaseCtrl = { +CaseCtrl gCaseCtrl = { // default .precision = TIME_PRECISION_MILLI, .bindNullNum = 0, .printCreateTblSql = false, @@ -2596,6 +2596,8 @@ void runAll(TAOS *taos) { printf("%s Begin\n", gCaseCtrl.caseCatalog); runCaseList(taos); +#if 0 + strcpy(gCaseCtrl.caseCatalog, "Micro DB precision Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.precision = TIME_PRECISION_MICRO; @@ -2626,7 +2628,6 @@ void runAll(TAOS *taos) { runCaseList(taos); gCaseCtrl.bindRowNum = 0; -#if 0 strcpy(gCaseCtrl.caseCatalog, "Row Num Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.rowNum = 1000; @@ -2640,7 +2641,6 @@ void runAll(TAOS *taos) { gCaseCtrl.runTimes = 2; runCaseList(taos); gCaseCtrl.runTimes = 0; -#endif strcpy(gCaseCtrl.caseCatalog, "Check Param Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); @@ -2648,7 +2648,6 @@ void runAll(TAOS *taos) { runCaseList(taos); gCaseCtrl.checkParamNum = false; -#if 0 strcpy(gCaseCtrl.caseCatalog, "Bind Col Num Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.bindColNum = 6; From 3e2ded3973917931cc84157c29884d3610fe5cf3 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 19 Jul 2022 10:12:02 +0800 Subject: [PATCH 03/47] fix: add debug info --- source/libs/executor/src/scanoperator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 200b920a0e..91aefd3406 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2020,8 +2020,8 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; int32_t code = metaGetTableEntryByUid(&mr, suid); if (code != TSDB_CODE_SUCCESS) { - qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), - GET_TASKID(pTaskInfo)); + qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", + pInfo->pCur->mr.me.name, suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); metaReaderClear(&mr); metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; From 80808766c17d30c25aa7f5febb903cb5057d2d53 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 19 Jul 2022 15:17:43 +0800 Subject: [PATCH 04/47] fix: fix memory leak --- source/client/inc/clientStmt.h | 2 +- source/client/src/clientStmt.c | 16 ++++++++++------ source/libs/parser/src/parInsertData.c | 1 + 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index c2b5d1de6f..a7adaef966 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -72,7 +72,6 @@ typedef struct SStmtBindInfo { typedef struct SStmtExecInfo { int32_t affectedRows; SRequestObj* pRequest; - SHashObj* pVgHash; SHashObj* pBlockHash; bool autoCreateTbl; } SStmtExecInfo; @@ -88,6 +87,7 @@ typedef struct SStmtSQLInfo { SArray* nodeList; SStmtQueryResInfo queryRes; bool autoCreateTbl; + SHashObj* pVgHash; } SStmtSQLInfo; typedef struct STscStmt { diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 7a83006961..70edb32f2d 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -160,7 +160,7 @@ int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash, bool autoCreateTbl) { STscStmt* pStmt = (STscStmt*)stmt; - pStmt->exec.pVgHash = pVgHash; + pStmt->sql.pVgHash = pVgHash; pStmt->exec.pBlockHash = pBlockHash; pStmt->exec.autoCreateTbl = autoCreateTbl; @@ -182,7 +182,7 @@ int32_t stmtUpdateInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, char int32_t stmtGetExecInfo(TAOS_STMT* stmt, SHashObj** pVgHash, SHashObj** pBlockHash) { STscStmt* pStmt = (STscStmt*)stmt; - *pVgHash = pStmt->exec.pVgHash; + *pVgHash = pStmt->sql.pVgHash; *pBlockHash = pStmt->exec.pBlockHash; return TSDB_CODE_SUCCESS; @@ -313,6 +313,8 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { taosMemoryFree(pStmt->sql.sqlStr); qDestroyQuery(pStmt->sql.pQuery); taosArrayDestroy(pStmt->sql.nodeList); + taosHashCleanup(pStmt->sql.pVgHash); + pStmt->sql.pVgHash = NULL; void* pIter = taosHashIterate(pStmt->sql.pTableCache, NULL); while (pIter) { @@ -345,7 +347,7 @@ int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataBlocks* pDataBlock, STab STMT_ERR_RET(catalogGetTableHashVgroup(pStmt->pCatalog, &conn, &pStmt->bInfo.sname, &vgInfo)); STMT_ERR_RET( - taosHashPut(pStmt->exec.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo))); + taosHashPut(pStmt->sql.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo))); STMT_ERR_RET(qRebuildStmtDataBlock(newBlock, pDataBlock, uid, vgInfo.vgId)); @@ -685,6 +687,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { if (pStmt->sql.pQuery->haveResultSet) { setResSchemaInfo(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->pResSchema, pStmt->sql.pQuery->numOfResCols); + taosMemoryFreeClear(pStmt->sql.pQuery->pResSchema); setResPrecision(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->precision); } @@ -809,7 +812,7 @@ int stmtExec(TAOS_STMT* stmt) { if (STMT_TYPE_QUERY == pStmt->sql.type) { launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); } else { - STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash)); + STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->exec.pBlockHash)); launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, (autoCreateTbl ? (void**)&pRsp : NULL)); } @@ -852,9 +855,10 @@ _return: int stmtClose(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_RET(stmtCleanSQLInfo(pStmt)); - + stmtCleanSQLInfo(pStmt); taosMemoryFree(stmt); + + return TSDB_CODE_SUCCESS; } const char* stmtErrstr(TAOS_STMT* stmt) { diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 290c65de12..9e1d8dba8b 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -678,6 +678,7 @@ void qFreeStmtDataBlock(void* pDataBlock) { return; } + taosMemoryFreeClear(((STableDataBlocks*)pDataBlock)->pTableMeta); taosMemoryFreeClear(((STableDataBlocks*)pDataBlock)->pData); taosMemoryFreeClear(pDataBlock); } From d2e780b14e778989dd54ab28c8b2381a998a6d92 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 19 Jul 2022 16:47:53 +0800 Subject: [PATCH 05/47] fix: fix memory leak --- source/libs/transport/src/trans.c | 2 +- source/libs/transport/src/transComm.c | 3 ++- tests/script/api/batchprepare.c | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 725f3b32cf..79f2f17a6e 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -112,7 +112,7 @@ void* rpcMallocCont(int32_t contLen) { void rpcFreeCont(void* cont) { if (cont == NULL) return; taosMemoryFree((char*)cont - TRANS_MSG_OVERHEAD); - tTrace("free mem:%p", (char*)cont - TRANS_MSG_OVERHEAD); + tTrace("rpc free cont:%p", (char*)cont - TRANS_MSG_OVERHEAD); } void* rpcReallocCont(void* ptr, int32_t contLen) { diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 84af8da513..0e1e4c6040 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -124,6 +124,7 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { SConnBuffer* p = connBuf; if (p->cap == 0) { p->buf = (char*)taosMemoryCalloc(CAPACITY, sizeof(char)); + tTrace("internal malloc mem:%p, size:%d", p->buf, CAPACITY); p->len = 0; p->cap = CAPACITY; p->total = -1; @@ -136,7 +137,7 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { } else { p->cap = p->total; p->buf = taosMemoryRealloc(p->buf, p->cap); - tTrace("internal malloc mem:%p, size:%d", p->buf, p->cap); + tTrace("internal realloc mem:%p, size:%d", p->buf, p->cap); uvBuf->base = p->buf + p->len; uvBuf->len = p->cap - p->len; diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index a330b6416e..e89a8b33eb 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -328,7 +328,7 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper //.optrIdxList = optrIdxList, //.bindColTypeNum = tListLen(bindColTypeList), //.bindColTypeList = bindColTypeList, - .caseIdx = 24, + .caseIdx = 8, .caseNum = 1, .caseRunNum = 1, }; @@ -1384,6 +1384,7 @@ void bpCheckTagFields(TAOS_STMT *stmt, TAOS_MULTI_BIND* pBind) { } bpCheckColTagFields(stmt, fieldNum, pFields, gCurCase->bindTagNum, pBind, BP_BIND_TAG); + taosMemoryFree(pFields); } void bpCheckColFields(TAOS_STMT *stmt, TAOS_MULTI_BIND* pBind) { @@ -1401,6 +1402,7 @@ void bpCheckColFields(TAOS_STMT *stmt, TAOS_MULTI_BIND* pBind) { } bpCheckColTagFields(stmt, fieldNum, pFields, gCurCase->bindColNum, pBind, BP_BIND_COL); + taosMemoryFree(pFields); } void bpShowBindParam(TAOS_MULTI_BIND *bind, int32_t num) { From c0a21dc911978a0421751bf25fd688b6c16665f0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 19 Jul 2022 16:56:37 +0800 Subject: [PATCH 06/47] avoid mem leak --- source/libs/transport/src/transCli.c | 3 +++ source/libs/transport/src/transComm.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index be3111e870..5d63c8daf6 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -198,6 +198,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { } \ destroyCmsg(pMsg); \ cliReleaseUnfinishedMsg(conn); \ + transQueueClear(&conn->cliMsgs); \ addConnToPool(((SCliThrd*)conn->hostThrd)->pool, conn); \ return; \ } \ @@ -545,6 +546,7 @@ static void addConnToPool(void* pool, SCliConn* conn) { STrans* pTransInst = thrd->pTransInst; conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); + cliReleaseUnfinishedMsg(conn); transQueueClear(&conn->cliMsgs); transCtxCleanup(&conn->ctx); conn->status = ConnInPool; @@ -645,6 +647,7 @@ static void cliDestroy(uv_handle_t* handle) { conn->stream->data = NULL; taosMemoryFree(conn->stream); transCtxCleanup(&conn->ctx); + cliReleaseUnfinishedMsg(conn); transQueueDestroy(&conn->cliMsgs); tTrace("%s conn %p destroy successfully", CONN_GET_INST_LABEL(conn), conn); transReqQueueClear(&conn->wreqQueue); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 84af8da513..e74f64faae 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -240,7 +240,7 @@ void transCtxCleanup(STransCtx* ctx) { ctx->freeFunc(iter->val); iter = taosHashIterate(ctx->args, iter); } - + ctx->freeFunc(ctx->brokenVal.val); taosHashCleanup(ctx->args); ctx->args = NULL; } From d266bfe1875e758bcb33a23243fb69d1b022dcb0 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 20 Jul 2022 08:39:52 +0800 Subject: [PATCH 07/47] fix: fix stmt memory leak --- include/os/osSysinfo.h | 2 +- source/dnode/mgmt/exe/dmMain.c | 4 ++-- source/libs/function/src/udfd.c | 4 ++-- source/os/src/osSysinfo.c | 2 +- tests/script/api/batchprepare.c | 9 +++++---- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 4ec2e2884e..6eed31b5e9 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -33,7 +33,7 @@ typedef struct { SDiskSize size; } SDiskSpace; -bool taosCheckSystemIsSmallEnd(); +bool taosCheckSystemIsLittleEnd(); void taosGetSystemInfo(); int32_t taosGetEmail(char *email, int32_t maxLen); int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen); diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 00c32e1990..013cc05c65 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -158,8 +158,8 @@ static void taosCleanupArgs() { } int main(int argc, char const *argv[]) { - if (!taosCheckSystemIsSmallEnd()) { - printf("failed to start since on non-small-end machines\n"); + if (!taosCheckSystemIsLittleEnd()) { + printf("failed to start since on non-little-end machines\n"); return -1; } diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 74fca69aa7..2402607251 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -913,8 +913,8 @@ void udfdConnectMnodeThreadFunc(void *args) { } int main(int argc, char *argv[]) { - if (!taosCheckSystemIsSmallEnd()) { - printf("failed to start since on non-small-end machines\n"); + if (!taosCheckSystemIsLittleEnd()) { + printf("failed to start since on non-little-end machines\n"); return -1; } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index b6220b0ae8..8450e8baea 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -210,7 +210,7 @@ static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { } -bool taosCheckSystemIsSmallEnd() { +bool taosCheckSystemIsLittleEnd() { union check { int16_t i; char ch[2]; diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index e89a8b33eb..ada2039460 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -299,7 +299,7 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper .printRes = true, .runTimes = 0, .caseRunIdx = -1, - .caseIdx = 23, + .caseIdx = 5, .caseNum = 1, .caseRunNum = 1, }; @@ -1408,7 +1408,7 @@ void bpCheckColFields(TAOS_STMT *stmt, TAOS_MULTI_BIND* pBind) { void bpShowBindParam(TAOS_MULTI_BIND *bind, int32_t num) { for (int32_t i = 0; i < num; ++i) { TAOS_MULTI_BIND* b = &bind[i]; - printf("Bind %d: type[%d],buf[%p],buflen[%d],len[%],null[%d],num[%d]\n", + printf("Bind %d: type[%d],buf[%p],buflen[%d],len[%d],null[%d],num[%d]\n", i, b->buffer_type, b->buffer, b->buffer_length, b->length ? *b->length : 0, b->is_null ? *b->is_null : 0, b->num); } } @@ -2599,7 +2599,6 @@ void runAll(TAOS *taos) { runCaseList(taos); #if 0 - strcpy(gCaseCtrl.caseCatalog, "Micro DB precision Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.precision = TIME_PRECISION_MICRO; @@ -2655,13 +2654,15 @@ void runAll(TAOS *taos) { gCaseCtrl.bindColNum = 6; runCaseList(taos); gCaseCtrl.bindColNum = 0; +#endif +/* strcpy(gCaseCtrl.caseCatalog, "Bind Col Type Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.bindColTypeNum = tListLen(bindColTypeList); gCaseCtrl.bindColTypeList = bindColTypeList; runCaseList(taos); -#endif +*/ printf("All Test End\n"); } From 5ed719c727241cdc6b0e36d641b514738043371c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 20 Jul 2022 09:39:58 +0800 Subject: [PATCH 08/47] fix: fix catalog memory leak --- include/common/tname.h | 1 + include/libs/qcom/query.h | 2 ++ source/common/src/tname.c | 5 +++++ source/libs/catalog/inc/catalogInt.h | 2 -- source/libs/catalog/src/catalog.c | 6 +++--- source/libs/catalog/src/ctgCache.c | 14 ++++++------- source/libs/catalog/src/ctgRemote.c | 26 +++++++++++++++++++++++++ source/libs/executor/src/scanoperator.c | 3 ++- 8 files changed, 46 insertions(+), 13 deletions(-) diff --git a/include/common/tname.h b/include/common/tname.h index 77965947ad..89c7764404 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -50,6 +50,7 @@ bool tNameIsValid(const SName* name); const char* tNameGetTableName(const SName* name); int32_t tNameGetDbName(const SName* name, char* dst); +const char* tNameGetDbNameP(const SName* name); int32_t tNameGetFullDbName(const SName* name, char* dst); diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 58739b4af7..4efcc9031b 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -260,6 +260,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define REQUEST_TOTAL_EXEC_TIMES 2 +#define IS_SYS_DBNAME(_dbname) (((*(_dbname) == 'i') && (0 == strcmp(_dbname, TSDB_INFORMATION_SCHEMA_DB))) || ((*(_dbname) == 'p') && (0 == strcmp(_dbname, TSDB_PERFORMANCE_SCHEMA_DB)))) + #define qFatal(...) \ do { \ if (qDebugFlag & DEBUG_FATAL) { \ diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 7183153824..c5bebf3630 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -190,6 +190,11 @@ int32_t tNameGetDbName(const SName* name, char* dst) { return 0; } +const char* tNameGetDbNameP(const SName* name) { + return &name->dbname[0]; +} + + int32_t tNameGetFullDbName(const SName* name, char* dst) { assert(name != NULL && dst != NULL); snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 9003de97d7..bf3bc1f0f4 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -460,8 +460,6 @@ typedef struct SCtgOperation { #define CTG_FLAG_MAKE_STB(_isStb) (((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB)) #define CTG_FLAG_MATCH_STB(_flag, tbType) (CTG_FLAG_IS_UNKNOWN_STB(_flag) || (CTG_FLAG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || (CTG_FLAG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE)) -#define CTG_IS_SYS_DBNAME(_dbname) (((*(_dbname) == 'i') && (0 == strcmp(_dbname, TSDB_INFORMATION_SCHEMA_DB))) || ((*(_dbname) == 'p') && (0 == strcmp(_dbname, TSDB_PERFORMANCE_SCHEMA_DB)))) - #define CTG_META_SIZE(pMeta) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) #define CTG_TABLE_NOT_EXIST(code) (code == CTG_ERR_CODE_TABLE_NOT_EXIST) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 1b7f53ae67..59f11898fa 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -865,7 +865,7 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo *pConn, SArray* tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - if (CTG_IS_SYS_DBNAME(name.dbname)) { + if (IS_SYS_DBNAME(name.dbname)) { continue; } @@ -936,7 +936,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - if (CTG_IS_SYS_DBNAME(pTableName->dbname)) { + if (IS_SYS_DBNAME(pTableName->dbname)) { ctgError("no valid vgInfo for db, dbname:%s", pTableName->dbname); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -947,7 +947,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const int32_t catalogGetTableHashVgroup(SCatalog *pCtg, SRequestConnInfo *pConn, const SName *pTableName, SVgroupInfo *pVgroup) { CTG_API_ENTER(); - if (CTG_IS_SYS_DBNAME(pTableName->dbname)) { + if (IS_SYS_DBNAME(pTableName->dbname)) { ctgError("no valid vgInfo for db, dbname:%s", pTableName->dbname); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 499ce77276..06e8216e87 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -132,7 +132,7 @@ void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { char *p = strchr(dbFName, '.'); - if (p && CTG_IS_SYS_DBNAME(p + 1)) { + if (p && IS_SYS_DBNAME(p + 1)) { dbFName = p + 1; } @@ -694,7 +694,7 @@ int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) } char *p = strchr(dbFName, '.'); - if (p && CTG_IS_SYS_DBNAME(p + 1)) { + if (p && IS_SYS_DBNAME(p + 1)) { dbFName = p + 1; } @@ -727,7 +727,7 @@ int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char *dbFName, bool syncOp) } char *p = strchr(dbFName, '.'); - if (p && CTG_IS_SYS_DBNAME(p + 1)) { + if (p && IS_SYS_DBNAME(p + 1)) { dbFName = p + 1; } @@ -823,7 +823,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId } char *p = strchr(dbFName, '.'); - if (p && CTG_IS_SYS_DBNAME(p + 1)) { + if (p && IS_SYS_DBNAME(p + 1)) { dbFName = p + 1; } @@ -859,7 +859,7 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool sy } char *p = strchr(output->dbFName, '.'); - if (p && CTG_IS_SYS_DBNAME(p + 1)) { + if (p && IS_SYS_DBNAME(p + 1)) { memmove(output->dbFName, p + 1, strlen(p + 1)); } @@ -2123,7 +2123,7 @@ int32_t ctgStartUpdateThread() { int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { - if (CTG_IS_SYS_DBNAME(ctx->pName->dbname)) { + if (IS_SYS_DBNAME(ctx->pName->dbname)) { CTG_FLAG_SET_SYS_DB(ctx->flag); } @@ -2177,7 +2177,7 @@ _return: } int32_t ctgGetTbHashVgroupFromCache(SCatalog *pCtg, const SName *pTableName, SVgroupInfo **pVgroup) { - if (CTG_IS_SYS_DBNAME(pTableName->dbname)) { + if (IS_SYS_DBNAME(pTableName->dbname)) { ctgError("no valid vgInfo for db, dbname:%s", pTableName->dbname); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 1e375471f9..cc5dde9298 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -375,6 +375,8 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL)); + rpcFreeCont(rpcRsp.pCont); + return TSDB_CODE_SUCCESS; } @@ -408,6 +410,8 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL)); + rpcFreeCont(rpcRsp.pCont); + return TSDB_CODE_SUCCESS; } @@ -447,6 +451,8 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildU CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, input->db)); + rpcFreeCont(rpcRsp.pCont); + return TSDB_CODE_SUCCESS; } @@ -485,6 +491,8 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)dbFName)); + rpcFreeCont(rpcRsp.pCont); + return TSDB_CODE_SUCCESS; } @@ -522,6 +530,8 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)indexName)); + + rpcFreeCont(rpcRsp.pCont); return TSDB_CODE_SUCCESS; } @@ -563,6 +573,8 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *n rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); + + rpcFreeCont(rpcRsp.pCont); return TSDB_CODE_SUCCESS; } @@ -602,6 +614,8 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const ch CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)funcName)); + rpcFreeCont(rpcRsp.pCont); + return TSDB_CODE_SUCCESS; } @@ -639,6 +653,8 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)user)); + + rpcFreeCont(rpcRsp.pCont); return TSDB_CODE_SUCCESS; } @@ -683,6 +699,8 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, tbFName)); + rpcFreeCont(rpcRsp.pCont); + return TSDB_CODE_SUCCESS; } @@ -740,6 +758,8 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SNa CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, tbFName)); + rpcFreeCont(rpcRsp.pCont); + return TSDB_CODE_SUCCESS; } @@ -784,6 +804,8 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const S rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); + + rpcFreeCont(rpcRsp.pCont); return TSDB_CODE_SUCCESS; } @@ -824,6 +846,8 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const S rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); + + rpcFreeCont(rpcRsp.pCont); return TSDB_CODE_SUCCESS; } @@ -858,6 +882,8 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, char **ou rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL)); + + rpcFreeCont(rpcRsp.pCont); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index bc526f8a31..48e6f51e42 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2147,11 +2147,12 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { SSysTableScanInfo* pInfo = pOperator->info; const char* name = tNameGetTableName(&pInfo->name); + const char* dbName = tNameGetDbNameP(&pInfo->name); if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { return sysTableScanUserTables(pOperator); } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { return sysTableScanUserTags(pOperator); - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, TSDB_TABLE_FNAME_LEN) == 0) { + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && IS_SYS_DBNAME(dbName)) { return sysTableScanUserSTables(pOperator); } else { // load the meta from mnode of the given epset if (pOperator->status == OP_EXEC_DONE) { From d6afa5e2c7137a5d05fb9830047d2150bb338a45 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 20 Jul 2022 09:46:53 +0800 Subject: [PATCH 09/47] fix: fix show stables issue --- source/libs/executor/src/scanoperator.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 48e6f51e42..6dc998e078 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2147,12 +2147,17 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { SSysTableScanInfo* pInfo = pOperator->info; const char* name = tNameGetTableName(&pInfo->name); - const char* dbName = tNameGetDbNameP(&pInfo->name); + if (pInfo->showRewrite) { + char dbName[TSDB_DB_NAME_LEN] = {0}; + getDBNameFromCondition(pInfo->pCondition, dbName); + sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); + } + if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { return sysTableScanUserTables(pOperator); } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { return sysTableScanUserTags(pOperator); - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && IS_SYS_DBNAME(dbName)) { + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && IS_SYS_DBNAME(pInfo->req.db)) { return sysTableScanUserSTables(pOperator); } else { // load the meta from mnode of the given epset if (pOperator->status == OP_EXEC_DONE) { @@ -2164,12 +2169,6 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); strcpy(pInfo->req.user, pInfo->pUser); - if (pInfo->showRewrite) { - char dbName[TSDB_DB_NAME_LEN] = {0}; - getDBNameFromCondition(pInfo->pCondition, dbName); - sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); - } - int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); char* buf1 = taosMemoryCalloc(1, contLen); tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req); From e477538f78398327ff64e7c5e8f539d4da15abf9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 20 Jul 2022 09:50:42 +0800 Subject: [PATCH 10/47] fix: keep one case --- tests/script/api/batchprepare.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index ada2039460..e1aa1991a4 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -250,7 +250,7 @@ CaseCtrl gCaseCtrl = { #endif -#if 1 +#if 0 CaseCtrl gCaseCtrl = { // default .precision = TIME_PRECISION_MILLI, .bindNullNum = 0, @@ -282,7 +282,7 @@ CaseCtrl gCaseCtrl = { // default }; #endif -#if 0 +#if 1 CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindNullNum = 1, .printCreateTblSql = false, From d60905aeb327c96a9f6182e4be434627ed60bfa6 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 20 Jul 2022 10:01:29 +0800 Subject: [PATCH 11/47] fix: fix mem leak --- source/client/src/clientImpl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 2ddf843a08..c83b80dcfc 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -621,6 +621,7 @@ int32_t buildSyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* } taosArrayPush(pDbVgList, &pVgList); + taosArrayDestroy(pVgList); } } From 4d18894657a32e27af0bd7a0c41636b3afb68323 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 20 Jul 2022 13:30:37 +0800 Subject: [PATCH 12/47] fix: fix mem leak --- source/client/src/clientImpl.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c83b80dcfc..ee78338a4e 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -590,6 +590,11 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray return code; } +void freeVgList(void *list) { + SArray* pList = *(SArray**)list; + taosArrayDestroy(pList); +} + int32_t buildSyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList) { SArray* pDbVgList = NULL; SArray* pQnodeList = NULL; @@ -621,7 +626,6 @@ int32_t buildSyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* } taosArrayPush(pDbVgList, &pVgList); - taosArrayDestroy(pVgList); } } @@ -642,7 +646,7 @@ int32_t buildSyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* _return: - taosArrayDestroy(pDbVgList); + taosArrayDestroyEx(pDbVgList, freeVgList); taosArrayDestroy(pQnodeList); return code; From 38431432acb1f0cc22cc9725905ea88e47af7151 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 20 Jul 2022 13:48:14 +0800 Subject: [PATCH 13/47] fix: avoid rpc mem leak --- source/libs/transport/src/transCli.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 5d63c8daf6..efb2434779 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1243,7 +1243,10 @@ void transReleaseCliHandle(void* handle) { void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); - if (pTransInst == NULL) return; + if (pTransInst == NULL) { + transFreeMsg(pReq->pCont); + return; + } bool valid = false; SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid); @@ -1282,7 +1285,10 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); - if (pTransInst == NULL) return; + if (pTransInst == NULL) { + transFreeMsg(pReq->pCont); + return; + } bool valid = false; SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid); From 7a62409dc171dd1afeb174c3dd0e38022b984157 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 20 Jul 2022 13:54:06 +0800 Subject: [PATCH 14/47] fix: fix taosd mem leak --- source/libs/executor/src/executorimpl.c | 6 +++--- source/libs/executor/src/scanoperator.c | 1 + source/libs/index/src/indexFilter.c | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d3a7fc51eb..885c6d7458 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4284,7 +4284,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, REPLACE_NODE(pNew); } else { taosMemoryFree(keyBuf); - nodesClearList(groupNew); + nodesDestroyList(groupNew); metaReaderClear(&mr); return code; } @@ -4302,7 +4302,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, if (tTagIsJson(data)) { terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; taosMemoryFree(keyBuf); - nodesClearList(groupNew); + nodesDestroyList(groupNew); metaReaderClear(&mr); return terrno; } @@ -4325,7 +4325,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, info->groupId = groupId; groupNum++; - nodesClearList(groupNew); + nodesDestroyList(groupNew); metaReaderClear(&mr); } taosMemoryFree(keyBuf); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6dc998e078..c78788aa5c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -359,6 +359,7 @@ void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* p SScalarParam param = {.columnData = pColInfoData}; fpSet.process(&srcParam, 1, ¶m); + colDataDestroy(&infoData); } static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index eadccba35f..27c90af3e7 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -707,6 +707,8 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { sifFreeParam(res); taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); } + sifFreeRes(ctx.pRes); + SIF_RET(code); } From 31cfa1fa5b55fbc5e0f71c40da523543cd45c546 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 20 Jul 2022 14:14:46 +0800 Subject: [PATCH 15/47] fix: avoid rpc mem leak --- include/libs/transport/trpc.h | 18 +++++++--------- source/libs/transport/inc/transComm.h | 14 ++++++------ source/libs/transport/src/trans.c | 31 +++++++++++---------------- source/libs/transport/src/transCli.c | 29 ++++++++++++++----------- source/libs/transport/src/transSvr.c | 23 ++++++++++---------- 5 files changed, 56 insertions(+), 59 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 2ae1f7b854..50f9959177 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -124,18 +124,16 @@ void *rpcReallocCont(void *ptr, int32_t contLen); // Because taosd supports multi-process mode // These functions should not be used on the server side // Please use tmsg functions, which are defined in tmsgcb.h -void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); -void rpcSendResponse(const SRpcMsg *pMsg); -void rpcRegisterBrokenLinkArg(SRpcMsg *msg); -void rpcReleaseHandle(void *handle, int8_t type); // just release conn to rpc instance, no close sock +int rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); +int rpcSendResponse(const SRpcMsg *pMsg); +int rpcRegisterBrokenLinkArg(SRpcMsg *msg); +int rpcReleaseHandle(void *handle, int8_t type); // just release conn to rpc instance, no close sock // These functions will not be called in the child process -void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet); -void rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx); -int32_t rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); -void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); -void rpcSetDefaultAddr(void *thandle, const char *ip, const char *fqdn); -void* rpcAllocHandle(); +int rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx); +int rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); +int rpcSetDefaultAddr(void *thandle, const char *ip, const char *fqdn); +void *rpcAllocHandle(); #ifdef __cplusplus } diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 2972f512f1..8af3c8b7fd 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -289,14 +289,14 @@ void transUnrefSrvHandle(void* handle); void transRefCliHandle(void* handle); void transUnrefCliHandle(void* handle); -void transReleaseCliHandle(void* handle); -void transReleaseSrvHandle(void* handle); +int transReleaseCliHandle(void* handle); +int transReleaseSrvHandle(void* handle); -void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransCtx* pCtx); -void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp); -void transSendResponse(const STransMsg* msg); -void transRegisterMsg(const STransMsg* msg); -void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn); +int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransCtx* pCtx); +int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp); +int transSendResponse(const STransMsg* msg); +int transRegisterMsg(const STransMsg* msg); +int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn); int64_t transAllocHandle(); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 79f2f17a6e..7633820292 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -25,7 +25,7 @@ void (*taosCloseHandle[])(void* arg) = {transCloseServer, transCloseClient}; void (*taosRefHandle[])(void* handle) = {transRefSrvHandle, transRefCliHandle}; void (*taosUnRefHandle[])(void* handle) = {transUnrefSrvHandle, transUnrefCliHandle}; -void (*transReleaseHandle[])(void* handle) = {transReleaseSrvHandle, transReleaseCliHandle}; +int (*transReleaseHandle[])(void* handle) = {transReleaseSrvHandle, transReleaseCliHandle}; static int32_t transValidLocalFqdn(const char* localFqdn, uint32_t* ip) { *ip = taosGetIpv4FromFqdn(localFqdn); @@ -129,25 +129,20 @@ void* rpcReallocCont(void* ptr, int32_t contLen) { return st + TRANS_MSG_OVERHEAD; } -void rpcSendRedirectRsp(void* thandle, const SEpSet* pEpSet) { - // deprecated api - assert(0); -} - int32_t rpcReportProgress(void* pConn, char* pCont, int32_t contLen) { return -1; } void rpcCancelRequest(int64_t rid) { return; } -void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { - transSendRequest(shandle, pEpSet, pMsg, NULL); +int rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { + return transSendRequest(shandle, pEpSet, pMsg, NULL); } -void rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) { - transSendRequest(shandle, pEpSet, pMsg, pCtx); +int rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) { + return transSendRequest(shandle, pEpSet, pMsg, pCtx); } -void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { - transSendRecv(shandle, pEpSet, pMsg, pRsp); +int rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { + return transSendRecv(shandle, pEpSet, pMsg, pRsp); } -void rpcSendResponse(const SRpcMsg* pMsg) { transSendResponse(pMsg); } +int rpcSendResponse(const SRpcMsg* pMsg) { return transSendResponse(pMsg); } void rpcRefHandle(void* handle, int8_t type) { assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT); @@ -159,15 +154,15 @@ void rpcUnrefHandle(void* handle, int8_t type) { (*taosUnRefHandle[type])(handle); } -void rpcRegisterBrokenLinkArg(SRpcMsg* msg) { transRegisterMsg(msg); } -void rpcReleaseHandle(void* handle, int8_t type) { +int rpcRegisterBrokenLinkArg(SRpcMsg* msg) { return transRegisterMsg(msg); } +int rpcReleaseHandle(void* handle, int8_t type) { assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT); - (*transReleaseHandle[type])(handle); + return (*transReleaseHandle[type])(handle); } -void rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { +int rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { // later - transSetDefaultAddr(thandle, ip, fqdn); + return transSetDefaultAddr(thandle, ip, fqdn); } void* rpcAllocHandle() { return (void*)transAllocHandle(); } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index efb2434779..0a4b7ed9ab 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1224,13 +1224,13 @@ SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle, bool* validHandle) { } return pThrd; } -void transReleaseCliHandle(void* handle) { +int transReleaseCliHandle(void* handle) { int idx = -1; bool valid = false; SCliThrd* pThrd = transGetWorkThrdFromHandle((int64_t)handle, &valid); if (pThrd == NULL) { - return; + return -1; } STransMsg tmsg = {.info.handle = handle}; SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg)); @@ -1238,14 +1238,14 @@ void transReleaseCliHandle(void* handle) { cmsg->type = Release; transAsyncSend(pThrd->asyncPool, &cmsg->q); - return; + return 0; } -void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { +int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); if (pTransInst == NULL) { transFreeMsg(pReq->pCont); - return; + return -1; } bool valid = false; @@ -1253,7 +1253,7 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra if (pThrd == NULL && valid == false) { transFreeMsg(pReq->pCont); transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); - return; + return -1; } TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); @@ -1280,14 +1280,14 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); ASSERT(transAsyncSend(pThrd->asyncPool, &(cliMsg->q)) == 0); transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); - return; + return 0; } -void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { +int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); if (pTransInst == NULL) { transFreeMsg(pReq->pCont); - return; + return -1; } bool valid = false; @@ -1295,7 +1295,7 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM if (pThrd == NULL && valid == false) { transFreeMsg(pReq->pCont); transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); - return; + return -1; } tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t)); @@ -1328,14 +1328,16 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM taosMemoryFree(sem); transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); - return; + return 0; } /* * **/ -void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { +int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); - if (pTransInst == NULL) return; + if (pTransInst == NULL) { + return -1; + } SCvtAddr cvtAddr = {0}; if (ip != NULL && fqdn != NULL) { @@ -1358,6 +1360,7 @@ void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { transAsyncSend(thrd->asyncPool, &(cliMsg->q)); } transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + return 0; } int64_t transAllocHandle() { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 9a511adf9b..7b9402f954 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1034,7 +1034,7 @@ void transUnrefSrvHandle(void* handle) { } } -void transReleaseSrvHandle(void* handle) { +int transReleaseSrvHandle(void* handle) { SRpcHandleInfo* info = handle; SExHandle* exh = info->handle; int64_t refId = info->refId; @@ -1053,16 +1053,16 @@ void transReleaseSrvHandle(void* handle) { tTrace("%s conn %p start to release", transLabel(pThrd->pTransInst), exh->handle); transAsyncSend(pThrd->asyncPool, &m->q); transReleaseExHandle(transGetRefMgt(), refId); - return; + return 0; _return1: tTrace("handle %p failed to send to release handle", exh); transReleaseExHandle(transGetRefMgt(), refId); - return; + return -1; _return2: tTrace("handle %p failed to send to release handle", exh); - return; + return -1; } -void transSendResponse(const STransMsg* msg) { +int transSendResponse(const STransMsg* msg) { SExHandle* exh = msg->info.handle; int64_t refId = msg->info.refId; ASYNC_CHECK_HANDLE(exh, refId); @@ -1082,18 +1082,18 @@ void transSendResponse(const STransMsg* msg) { tGTrace("conn %p start to send resp (1/2)", exh->handle); transAsyncSend(pThrd->asyncPool, &m->q); transReleaseExHandle(transGetRefMgt(), refId); - return; + return 0; _return1: tTrace("handle %p failed to send resp", exh); rpcFreeCont(msg->pCont); transReleaseExHandle(transGetRefMgt(), refId); - return; + return -1; _return2: tTrace("handle %p failed to send resp", exh); rpcFreeCont(msg->pCont); - return; + return -1; } -void transRegisterMsg(const STransMsg* msg) { +int transRegisterMsg(const STransMsg* msg) { SExHandle* exh = msg->info.handle; int64_t refId = msg->info.refId; ASYNC_CHECK_HANDLE(exh, refId); @@ -1112,16 +1112,17 @@ void transRegisterMsg(const STransMsg* msg) { tTrace("%s conn %p start to register brokenlink callback", transLabel(pTransInst), exh->handle); transAsyncSend(pThrd->asyncPool, &m->q); transReleaseExHandle(transGetRefMgt(), refId); - return; + return 0; _return1: tTrace("handle %p failed to register brokenlink", exh); rpcFreeCont(msg->pCont); transReleaseExHandle(transGetRefMgt(), refId); - return; + return -1; _return2: tTrace("handle %p failed to register brokenlink", exh); rpcFreeCont(msg->pCont); + return -1; } int transGetConnInfo(void* thandle, STransHandleInfo* pConnInfo) { return -1; } From c764ffb14fcb895744f094ada01f77b034444746 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 20 Jul 2022 15:22:56 +0800 Subject: [PATCH 16/47] fix rpc mem leak --- source/client/src/clientEnv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 53a1bd2235..635d4bf2f9 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -88,7 +88,7 @@ void closeTransporter(SAppInstInfo *pAppInfo) { static bool clientRpcRfp(int32_t code, tmsg_t msgType) { if (NEED_REDIRECT_ERROR(code)) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || - msgType == TDMT_SCH_MERGE_FETCH) { + msgType == TDMT_SCH_MERGE_FETCH || msgType == TDMT_SCH_QUERY_HEARTBEAT) { return false; } return true; From 85684cebb8e0f0b2215e17944570e0751f021add Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 20 Jul 2022 18:48:27 +0800 Subject: [PATCH 17/47] fix: fix mem leak --- include/libs/qcom/query.h | 4 ++-- source/libs/qcom/src/queryUtil.c | 13 ++++++++----- source/libs/scheduler/src/schRemote.c | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 4efcc9031b..cc040594b1 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -193,7 +193,7 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); void destroySendMsgInfo(SMsgSendInfo* pMsgBody); -int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, +int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo, bool persistHandle, void* ctx); /** @@ -205,7 +205,7 @@ int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTra * @param pInfo * @return */ -int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo); +int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo); int32_t queryBuildUseDbOutput(SUseDbOutput* pOut, SUseDbRsp* usedbRsp); diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 6b1476fe46..d8fda57791 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -148,11 +148,12 @@ void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { taosMemoryFreeClear(pMsgBody); } -int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, +int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo, bool persistHandle, void* rpcCtx) { char* pMsg = rpcMallocCont(pInfo->msgInfo.len); if (NULL == pMsg) { qError("0x%" PRIx64 " msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType)); + destroySendMsgInfo(pInfo); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return terrno; } @@ -167,13 +168,15 @@ int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTra .info.persistHandle = persistHandle, .code = 0 }; - assert(pInfo->fp != NULL); TRACE_SET_ROOTID(&rpcMsg.info.traceId, pInfo->requestId); - rpcSendRequestWithCtx(pTransporter, epSet, &rpcMsg, pTransporterId, rpcCtx); - return TSDB_CODE_SUCCESS; + int code = rpcSendRequestWithCtx(pTransporter, epSet, &rpcMsg, pTransporterId, rpcCtx); + if (code) { + destroySendMsgInfo(pInfo); + } + return code; } -int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo) { +int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo) { return asyncSendMsgToServerExt(pTransporter, epSet, pTransporterId, pInfo, false, NULL); } diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 5452ca31a5..83ea510962 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -555,7 +555,7 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { *fp = schHandleCallback; break; case TDMT_SCH_DROP_TASK: - *fp = schHandleDropCallback; + //*fp = schHandleDropCallback; break; case TDMT_SCH_QUERY_HEARTBEAT: *fp = schHandleHbCallback; From db37fef8988d7914fe2ad93b6615c4f4b7c09043 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 21 Jul 2022 09:10:18 +0800 Subject: [PATCH 18/47] fix: fix crash issue --- source/libs/scheduler/src/schRemote.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 83ea510962..b794cb91f5 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -509,7 +509,7 @@ int32_t schGenerateCallBackInfo(SSchJob *pJob, SSchTask *pTask, void *msg, uint3 SMsgSendInfo *msgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == msgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } msgSendInfo->paramFreeFp = taosMemoryFree; @@ -535,8 +535,12 @@ int32_t schGenerateCallBackInfo(SSchJob *pJob, SSchTask *pTask, void *msg, uint3 _return: - destroySendMsgInfo(msgSendInfo); + if (msgSendInfo) { + destroySendMsgInfo(msgSendInfo); + } + taosMemoryFree(msg); + SCH_RET(code); } @@ -555,7 +559,7 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { *fp = schHandleCallback; break; case TDMT_SCH_DROP_TASK: - //*fp = schHandleDropCallback; + *fp = schHandleDropCallback; break; case TDMT_SCH_QUERY_HEARTBEAT: *fp = schHandleHbCallback; @@ -843,6 +847,7 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQuery int64_t transporterId = 0; code = asyncSendMsgToServerExt(trans->pTrans, epSet, &transporterId, pMsgSendInfo, persistHandle, ctx); + pMsgSendInfo = NULL; if (code) { SCH_ERR_JRET(code); } @@ -919,7 +924,9 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) { addr.epSet.numOfEps = 1; memcpy(&addr.epSet.eps[0], &nodeEpId->ep, sizeof(nodeEpId->ep)); - SCH_ERR_JRET(schAsyncSendMsg(NULL, NULL, &trans, &addr, msgType, msg, msgSize, true, &rpcCtx)); + code = schAsyncSendMsg(NULL, NULL, &trans, &addr, msgType, msg, msgSize, true, &rpcCtx); + msg = NULL; + SCH_ERR_JRET(code); return TSDB_CODE_SUCCESS; @@ -1087,9 +1094,10 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)}; - SCH_ERR_JRET( - schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL))); - + schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); + msg = NULL; + SCH_ERR_JRET(code); + if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY) { SCH_ERR_RET(schAppendTaskExecNode(pJob, pTask, addr, pTask->execId)); } From dac39371cd1130469f54cdeb561156366aec15db Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 21 Jul 2022 10:27:39 +0800 Subject: [PATCH 19/47] fix: fix drop task memory leak --- source/client/src/clientEnv.c | 2 +- source/libs/scheduler/src/schTask.c | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 635d4bf2f9..5b96729503 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -88,7 +88,7 @@ void closeTransporter(SAppInstInfo *pAppInfo) { static bool clientRpcRfp(int32_t code, tmsg_t msgType) { if (NEED_REDIRECT_ERROR(code)) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || - msgType == TDMT_SCH_MERGE_FETCH || msgType == TDMT_SCH_QUERY_HEARTBEAT) { + msgType == TDMT_SCH_MERGE_FETCH || msgType == TDMT_SCH_QUERY_HEARTBEAT || msgType == TDMT_SCH_DROP_TASK) { return false; } return true; diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 282e81bb5d..c40e56ab6f 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -102,14 +102,14 @@ int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) { } int32_t schAppendTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t execId) { - SSchNodeInfo nodeInfo = {.addr = *addr, .handle = NULL}; + SSchNodeInfo nodeInfo = {.addr = *addr, .handle = SCH_GET_TASK_HANDLE(pTask)}; if (taosHashPut(pTask->execNodes, &execId, sizeof(execId), &nodeInfo, sizeof(nodeInfo))) { SCH_TASK_ELOG("taosHashPut nodeInfo to execNodes failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("task execNode added, execId:%d", execId); + SCH_TASK_DLOG("task execNode added, execId:%d, handle:%p", execId, nodeInfo.handle); return TSDB_CODE_SUCCESS; } @@ -752,12 +752,18 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) { return; } + int32_t i = 0; SSchNodeInfo *nodeInfo = taosHashIterate(pTask->execNodes, NULL); while (nodeInfo) { - SCH_SET_TASK_HANDLE(pTask, nodeInfo->handle); - - schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK); + if (nodeInfo->handle) { + SCH_SET_TASK_HANDLE(pTask, nodeInfo->handle); + schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK); + SCH_TASK_DLOG("start to drop task's %dth execNode", i); + } else { + SCH_TASK_DLOG("no need to drop task %dth execNode", i); + } + ++i; nodeInfo = taosHashIterate(pTask->execNodes, nodeInfo); } From 113dfdde0ae70ad88412c39a9064b59e938af436 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 21 Jul 2022 10:38:36 +0800 Subject: [PATCH 20/47] fix: increase query thread number --- source/common/src/tglobal.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index db8afba409..76ec5b2d22 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -51,15 +51,15 @@ int32_t tsNumOfShmThreads = 1; int32_t tsNumOfRpcThreads = 1; int32_t tsNumOfCommitThreads = 2; int32_t tsNumOfTaskQueueThreads = 1; -int32_t tsNumOfMnodeQueryThreads = 2; +int32_t tsNumOfMnodeQueryThreads = 4; int32_t tsNumOfMnodeFetchThreads = 1; int32_t tsNumOfMnodeReadThreads = 1; -int32_t tsNumOfVnodeQueryThreads = 2; +int32_t tsNumOfVnodeQueryThreads = 4; int32_t tsNumOfVnodeStreamThreads = 2; int32_t tsNumOfVnodeFetchThreads = 4; int32_t tsNumOfVnodeWriteThreads = 2; int32_t tsNumOfVnodeSyncThreads = 2; -int32_t tsNumOfQnodeQueryThreads = 2; +int32_t tsNumOfQnodeQueryThreads = 4; int32_t tsNumOfQnodeFetchThreads = 4; int32_t tsNumOfSnodeSharedThreads = 2; int32_t tsNumOfSnodeUniqueThreads = 2; @@ -402,16 +402,16 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4); if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, 0) != 0) return -1; - tsNumOfMnodeQueryThreads = tsNumOfCores / 8; - tsNumOfMnodeQueryThreads = TRANGE(tsNumOfMnodeQueryThreads, 1, 4); + tsNumOfMnodeQueryThreads = tsNumOfCores * 2; + tsNumOfMnodeQueryThreads = TRANGE(tsNumOfMnodeQueryThreads, 4, 8); if (cfgAddInt32(pCfg, "numOfMnodeQueryThreads", tsNumOfMnodeQueryThreads, 1, 1024, 0) != 0) return -1; tsNumOfMnodeReadThreads = tsNumOfCores / 8; tsNumOfMnodeReadThreads = TRANGE(tsNumOfMnodeReadThreads, 1, 4); if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, 0) != 0) return -1; - tsNumOfVnodeQueryThreads = tsNumOfCores / 4; - tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 2); + tsNumOfVnodeQueryThreads = tsNumOfCores * 2; + tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 4); if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 1, 1024, 0) != 0) return -1; tsNumOfVnodeStreamThreads = tsNumOfCores / 4; @@ -430,8 +430,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfVnodeSyncThreads = TMAX(tsNumOfVnodeSyncThreads, 1); if (cfgAddInt32(pCfg, "numOfVnodeSyncThreads", tsNumOfVnodeSyncThreads, 1, 1024, 0) != 0) return -1; - tsNumOfQnodeQueryThreads = tsNumOfCores / 2; - tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 1); + tsNumOfQnodeQueryThreads = tsNumOfCores * 2; + tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 4); if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, 0) != 0) return -1; tsNumOfQnodeFetchThreads = tsNumOfCores / 2; From 57436e5ae26a7210bd1c80982e291d847c3f1fdc Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 21 Jul 2022 10:42:00 +0800 Subject: [PATCH 21/47] avoid mem leak --- source/libs/transport/inc/transComm.h | 1 + source/libs/transport/src/transCli.c | 20 ++++++++++++++++---- source/libs/transport/src/transComm.c | 4 +++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 9eb5135969..843798817d 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -226,6 +226,7 @@ typedef struct { int index; int nAsync; uv_async_t* asyncs; + int8_t stop; } SAsyncPool; SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 32d1e7140e..557ed548f4 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1020,6 +1020,7 @@ void cliSendQuit(SCliThrd* thrd) { SCliMsg* msg = taosMemoryCalloc(1, sizeof(SCliMsg)); msg->type = Quit; transAsyncSend(thrd->asyncPool, &msg->q); + atomic_store_8(&thrd->asyncPool->stop, 1); } void cliWalkCb(uv_handle_t* handle, void* arg) { if (!uv_is_closing(handle)) { @@ -1238,7 +1239,9 @@ int transReleaseCliHandle(void* handle) { cmsg->msg = tmsg; cmsg->type = Release; - transAsyncSend(pThrd->asyncPool, &cmsg->q); + if (0 != transAsyncSend(pThrd->asyncPool, &cmsg->q)) { + return -1; + } return 0; } @@ -1279,7 +1282,10 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran STraceId* trace = &pReq->info.traceId; tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); - ASSERT(transAsyncSend(pThrd->asyncPool, &(cliMsg->q)) == 0); + if (0 != transAsyncSend(pThrd->asyncPool, &(cliMsg->q))) { + destroyCmsg(cliMsg); + return -1; + } transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return 0; } @@ -1323,7 +1329,10 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); - transAsyncSend(pThrd->asyncPool, &(cliMsg->q)); + if (0 != transAsyncSend(pThrd->asyncPool, &cliMsg->q)) { + destroyCmsg(cliMsg); + return -1; + } tsem_wait(sem); tsem_destroy(sem); taosMemoryFree(sem); @@ -1358,7 +1367,10 @@ int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { SCliThrd* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[i]; tDebug("%s update epset at thread:%08" PRId64, pTransInst->label, thrd->pid); - transAsyncSend(thrd->asyncPool, &(cliMsg->q)); + if (transAsyncSend(thrd->asyncPool, &(cliMsg->q)) != 0) { + destroyCmsg(cliMsg); + return -1; + } } transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return 0; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 7c76f69f0c..c89bbd408b 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -177,7 +177,6 @@ int transSetConnOption(uv_tcp_t* stream) { SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) { SAsyncPool* pool = taosMemoryCalloc(1, sizeof(SAsyncPool)); - pool->index = 0; pool->nAsync = sz; pool->asyncs = taosMemoryCalloc(1, sizeof(uv_async_t) * pool->nAsync); @@ -207,6 +206,9 @@ void transDestroyAsyncPool(SAsyncPool* pool) { taosMemoryFree(pool); } int transAsyncSend(SAsyncPool* pool, queue* q) { + if (atomic_load_8(&pool->stop) == 1) { + return -1; + } int idx = pool->index; idx = idx % pool->nAsync; // no need mutex here From ed1c777b4eff5ae02b879f843fb2bf54b0a01a13 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 21 Jul 2022 11:09:29 +0800 Subject: [PATCH 22/47] fix: avoid rpc mem leak --- source/libs/transport/src/transCli.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 557ed548f4..07a698f883 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1284,6 +1284,7 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); if (0 != transAsyncSend(pThrd->asyncPool, &(cliMsg->q))) { destroyCmsg(cliMsg); + transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return -1; } transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); @@ -1330,7 +1331,10 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); if (0 != transAsyncSend(pThrd->asyncPool, &cliMsg->q)) { + tsem_destroy(sem); + taosMemoryFree(sem); destroyCmsg(cliMsg); + transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return -1; } tsem_wait(sem); @@ -1369,6 +1373,7 @@ int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { if (transAsyncSend(thrd->asyncPool, &(cliMsg->q)) != 0) { destroyCmsg(cliMsg); + transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return -1; } } From 77e481c7ef0d11be0b9232d50ebbddfd9dd315ac Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 21 Jul 2022 11:52:03 +0800 Subject: [PATCH 23/47] test: restore 2.0 case --- tests/script/jenkins/basic.txt | 31 ++++---- tests/script/tsim/parser/commit.sim | 5 +- tests/script/tsim/parser/condition.sim | 13 +--- tests/script/tsim/parser/condition_query.sim | 39 +++++----- tests/script/tsim/parser/create_mt.sim | 36 ++++----- tests/script/tsim/parser/create_tb.sim | 10 ++- tests/script/tsim/parser/dbtbnameValidate.sim | 73 +++++++++---------- tests/script/tsim/parser/distinct.sim | 3 +- 8 files changed, 93 insertions(+), 117 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 6c1938d9fc..70e5bdcfa2 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -91,24 +91,23 @@ ./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim ./test.sh -f tsim/parser/between_and.sim ./test.sh -f tsim/parser/binary_escapeCharacter.sim -# nojira ./test.sh -f tsim/parser/col_arithmetic_operation.sim -# nojira ./test.sh -f tsim/parser/columnValue.sim -## ./test.sh -f tsim/parser/commit.sim -## ./test.sh -f tsim/parser/condition.sim +# jira ./test.sh -f tsim/parser/col_arithmetic_operation.sim +# jira ./test.sh -f tsim/parser/columnValue.sim +./test.sh -f tsim/parser/commit.sim +# jira ./test.sh -f tsim/parser/condition.sim ## ./test.sh -f tsim/parser/constCol.sim -# ./test.sh -f tsim/parser/create_db.sim -## ./test.sh -f tsim/parser/create_db__for_community_version.sim -# ./test.sh -f tsim/parser/create_mt.sim -# ./test.sh -f tsim/parser/create_tb.sim -## ./test.sh -f tsim/parser/create_tb_with_tag_name.sim -# ./test.sh -f tsim/parser/dbtbnameValidate.sim -##./test.sh -f tsim/parser/distinct.sim -# ./test.sh -f tsim/parser/fill.sim -# ./test.sh -f tsim/parser/fill_stb.sim -## ./test.sh -f tsim/parser/fill_us.sim -# ./test.sh -f tsim/parser/first_last.sim +./test.sh -f tsim/parser/create_db.sim +./test.sh -f tsim/parser/create_mt.sim +# jira ./test.sh -f tsim/parser/create_tb_with_tag_name.sim +./test.sh -f tsim/parser/create_tb.sim +./test.sh -f tsim/parser/dbtbnameValidate.sim +./test.sh -f tsim/parser/distinct.sim +# jira ./test.sh -f tsim/parser/fill_stb.sim +./test.sh -f tsim/parser/fill_us.sim +./test.sh -f tsim/parser/fill.sim +./test.sh -f tsim/parser/first_last.sim ./test.sh -f tsim/parser/fourArithmetic-basic.sim -## ./test.sh -f tsim/parser/function.sim +# jira ./test.sh -f tsim/parser/function.sim ./test.sh -f tsim/parser/groupby-basic.sim # ./test.sh -f tsim/parser/groupby.sim ## ./test.sh -f tsim/parser/having.sim diff --git a/tests/script/tsim/parser/commit.sim b/tests/script/tsim/parser/commit.sim index 83b457673b..0877168609 100644 --- a/tests/script/tsim/parser/commit.sim +++ b/tests/script/tsim/parser/commit.sim @@ -20,7 +20,7 @@ $stb = $stbPrefix . $i sql drop database $db -x step1 step1: -sql create database $db maxrows 255 ctime 3600 +sql create database $db maxrows 255 print ====== create tables sql use $db sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int) @@ -78,12 +78,9 @@ endw print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 system sh/exec.sh -n dnode1 -s start -sleep 100 print ================== server restart completed sql connect -sleep 100 print ====== select from table and check num of rows returned sql use $db diff --git a/tests/script/tsim/parser/condition.sim b/tests/script/tsim/parser/condition.sim index 8c1327baae..700d1b98c0 100644 --- a/tests/script/tsim/parser/condition.sim +++ b/tests/script/tsim/parser/condition.sim @@ -2,11 +2,11 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect + sql drop database if exists cdb sql create database if not exists cdb sql use cdb sql create table stb1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double) - sql create table tb1 using stb1 tags(1,'1',1.0) sql create table tb2 using stb1 tags(2,'2',2.0) sql create table tb3 using stb1 tags(3,'3',3.0) @@ -45,7 +45,6 @@ sql insert into tb6 values ('2021-05-05 18:19:27',64,64.0,64,64,64,64.0,false,'6 sql insert into tb6 values ('2021-05-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) sql create table stb2 (ts timestamp, u1 int unsigned, u2 bigint unsigned, u3 smallint unsigned, u4 tinyint unsigned, ts2 timestamp) TAGS(t1 int unsigned, t2 bigint unsigned, t3 timestamp, t4 int) - sql create table tb2_1 using stb2 tags(1,1,'2021-05-05 18:38:38',1) sql create table tb2_2 using stb2 tags(2,2,'2021-05-05 18:58:58',2) @@ -67,7 +66,6 @@ sql insert into tb2_2 values ('2021-05-05 18:19:14',8,2,3,4,'2021-05-05 18:28:15 sql insert into tb2_2 values ('2021-05-05 18:19:15',5,6,7,8,'2021-05-05 18:28:16') sql create table stb3 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double) - sql create table tb3_1 using stb3 tags(1,'1',1.0) sql create table tb3_2 using stb3 tags(2,'2',2.0) @@ -78,7 +76,6 @@ sql insert into tb3_1 values ('2021-04-05 18:19:03',4,4.0,4,4,4,4.0,false,'4','4 sql insert into tb3_1 values ('2021-05-05 18:19:28',5,NULL,5,NULL,5,NULL,true,NULL,'5') sql insert into tb3_1 values ('2021-06-05 18:19:28',NULL,6.0,NULL,6,NULL,6.0,NULL,'6',NULL) sql insert into tb3_1 values ('2021-07-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) - sql insert into tb3_2 values ('2021-01-06 18:19:00',11,11.0,11,11,11,11.0,true ,'11','11') sql insert into tb3_2 values ('2021-02-06 18:19:01',12,12.0,12,12,12,12.0,true ,'12','12') sql insert into tb3_2 values ('2021-03-06 18:19:02',13,13.0,13,13,13,13.0,false,'13','13') @@ -87,9 +84,7 @@ sql insert into tb3_2 values ('2021-05-06 18:19:28',15,NULL,15,NULL,15,NULL,true sql insert into tb3_2 values ('2021-06-06 18:19:28',NULL,16.0,NULL,16,NULL,16.0,NULL,'16',NULL) sql insert into tb3_2 values ('2021-07-06 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) - sql create table stb4 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9),c10 binary(16300)) TAGS(t1 int, t2 binary(10), t3 double) - sql create table tb4_0 using stb4 tags(0,'0',0.0) sql create table tb4_1 using stb4 tags(1,'1',1.0) sql create table tb4_2 using stb4 tags(2,'2',2.0) @@ -128,19 +123,13 @@ while $i < $blockNum $ts0 = $ts0 + 259200000 endw -sleep 100 - -sql connect - run tsim/parser/condition_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 100 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 100 run tsim/parser/condition_query.sim diff --git a/tests/script/tsim/parser/condition_query.sim b/tests/script/tsim/parser/condition_query.sim index 8dfa8dae0c..dc5eed49be 100644 --- a/tests/script/tsim/parser/condition_query.sim +++ b/tests/script/tsim/parser/condition_query.sim @@ -11,14 +11,14 @@ if $rows != 28 then return -1 endi -sql_error select * from stb1 where c8 > 0 -sql_error select * from stb1 where c7 in (0,2,3,1); -sql_error select * from stb1 where c8 in (true); -sql_error select * from stb1 where c8 in (1,2); -sql_error select * from stb1 where t2 in (3.0); -sql_error select ts,c1,c7 from stb1 where c7 > false -sql_error select * from stb1 where c1 > NULL; -sql_error select * from stb1 where c1 = NULL; +sql select * from stb1 where c8 > 0 +sql select * from stb1 where c7 in (0,2,3,1); +sql select * from stb1 where c8 in (true); +sql select * from stb1 where c8 in (1,2); +sql select * from stb1 where t2 in (3.0); +sql select ts,c1,c7 from stb1 where c7 > false +sql select * from stb1 where c1 > NULL; +sql select * from stb1 where c1 = NULL; sql_error select * from stb1 where c1 LIKE '%1'; sql_error select * from stb1 where c2 LIKE '%1'; sql_error select * from stb1 where c3 LIKE '%1'; @@ -26,20 +26,20 @@ sql_error select * from stb1 where c4 LIKE '%1'; sql_error select * from stb1 where c5 LIKE '%1'; sql_error select * from stb1 where c6 LIKE '%1'; sql_error select * from stb1 where c7 LIKE '%1'; -sql_error select * from stb1 where c1 = 'NULL'; -sql_error select * from stb1 where c2 > 'NULL'; -sql_error select * from stb1 where c3 <> 'NULL'; -sql_error select * from stb1 where c4 != 'null'; -sql_error select * from stb1 where c5 >= 'null'; -sql_error select * from stb1 where c6 <= 'null'; -sql_error select * from stb1 where c7 < 'nuLl'; -sql_error select * from stb1 where c8 < 'nuLl'; -sql_error select * from stb1 where c9 > 'nuLl'; +sql select * from stb1 where c1 = 'NULL'; +sql select * from stb1 where c2 > 'NULL'; +sql select * from stb1 where c3 <> 'NULL'; +sql select * from stb1 where c4 != 'null'; +sql select * from stb1 where c5 >= 'null'; +sql select * from stb1 where c6 <= 'null'; +sql select * from stb1 where c7 < 'nuLl'; +sql select * from stb1 where c8 < 'nuLl'; +sql select * from stb1 where c9 > 'nuLl'; sql_error select * from (select * from stb1 where c7=true) a, (select * from stb1 where c1 > 30) b; sql_error select a.ts,a.c1,a.c8 from (select * from stb1 where c7=true) a, (select * from stb1 where c1 > 30) b where a.ts=b.ts and a.c1 > 50 or b.c1 < 60; sql_error select a.ts,a.c1,a.c8 from (select * from stb1 where c7=true) a, (select * from stb1 where c1 > 30) b where a.ts=b.ts and ((a.c1 > 50 and a.c1 < 60) or (b.c2 > 60)); -sql_error select * from stb1 where 'c2' is null; -sql_error select * from stb1 where 'c2' is not null; +sql select * from stb1 where 'c2' is null; +sql select * from stb1 where 'c2' is not null; sql select * from stb1 where c2 > 3.0 or c2 < 60; if $rows != 28 then @@ -173,7 +173,6 @@ if $data32 != 0 then return -1 endi - sql select ts,c1,c7 from stb1 where c7 = true if $rows != 14 then return -1 diff --git a/tests/script/tsim/parser/create_mt.sim b/tests/script/tsim/parser/create_mt.sim index fafee66c76..8f0c0e030b 100644 --- a/tests/script/tsim/parser/create_mt.sim +++ b/tests/script/tsim/parser/create_mt.sim @@ -69,7 +69,8 @@ sql_error create table $mt (ts $i_ts , col int) tags (tag1 int) sql_error create table $mt (ts timestamp, col $i_binary ) tags (tag1 int) sql_error create table $mt (ts timestamp, col $i_bigint ) tags (tag1 int) sql_error create table $mt (ts timestamp, col $i_smallint ) tags (tag1 int) -sql_error create table $mt (ts timestamp, col $i_binary2 ) tags (tag1 int) +sql create table $mt (ts timestamp, col $i_binary2 ) tags (tag1 int) +sql drop table $mt sql_error create table $mt (ts timestamp, col $i_tinyint ) tags (tag1 int) sql_error create table $mt (ts timestamp, col $i_nchar ) tags (tag1 int) @@ -101,7 +102,8 @@ sql_error create table $mt (ts timestamp, col int) tags (tag1 $i_binary ) sql_error create table $mt (ts timestamp, col int) tags (tag1 $i_bigint ) sql_error create table $mt (ts timestamp, col int) tags (tag1 $i_smallint ) sql_error create table $mt (ts timestamp, col int) tags (tag1 $i_tinyint ) -sql_error create table $mt (ts timestamp, col int) tags (tag1 $i_binary2 ) +sql create table $mt (ts timestamp, col int) tags (tag1 $i_binary2 ) +sql drop table $mt sql_error create table $mt (ts timestamp, col int) tags (tag1 $i_bool ) sql_error create table $mt (ts timestamp, col int) tags (tag1 $nchar ) # correct use of nchar in tags @@ -144,7 +146,8 @@ sql_error create table $mt (ts timestamp, col1 int) tags ( $ses int) sql_error create table $mt (ts timestamp, col1 int) tags ( $int int) sql_error create table $mt (ts timestamp, col1 int) tags ( $bint int) sql_error create table $mt (ts timestamp, col1 int) tags ( $binary int) -sql_error create table $mt (ts timestamp, col1 int) tags ( $str int) +sql create table $mt (ts timestamp, col1 int) tags ( $str int) +sql drop table $mt sql_error create table $mt (ts timestamp, col1 int) tags ( $tag int) sql_error create table $mt (ts timestamp, col1 int) tags ( $tags int) sql_error create table $mt (ts timestamp, col1 int) tags ( $sint int) @@ -162,8 +165,8 @@ sql create table $tb using $mt tags (-1) # -x ng_tag_v # return -1 #ng_tag_v: -sql select tg from $tb -if $data00 != -1 then +sql show tags from $tb +if $data05 != -1 then return -1 endi sql drop table $tb @@ -172,28 +175,21 @@ sql drop table $tb print create_mt.sim unmatched_tag_types sql reset query cache sql create table $tb using $mt tags ('123') -sql select tg from $tb -print data00 = $data00 -if $data00 != 123 then +sql show tags from $tb +print data05 = $data05 +if $data05 != 123 then return -1 endi sql drop table $tb + sql_error create table $tb using $mt tags (abc) #the case below might need more consideration sql_error create table $tb using $mt tags ('abc') sql drop table if exists $tb sql reset query cache -sql create table $tb using $mt tags (1e1) -sql select tg from $tb -if $data00 != 10 then - return -1 -endi -sql drop table $tb -sql create table $tb using $mt tags ('1e1') -sql select tg from $tb -if $data00 != 10 then - return -1 -endi +sql_error create table $tb using $mt tags (1e1) + +sql_error create table $tb using $mt tags ('1e1') sql_error create table $tb using $mt tags (2147483649) ## case: chinese_char_in_metric @@ -245,7 +241,7 @@ print chinese_char_in_metrics test passed sql drop database $db sql show databases -if $rows != 0 then +if $rows != 2 then return -1 endi diff --git a/tests/script/tsim/parser/create_tb.sim b/tests/script/tsim/parser/create_tb.sim index 5203f289dc..384c8f8757 100644 --- a/tests/script/tsim/parser/create_tb.sim +++ b/tests/script/tsim/parser/create_tb.sim @@ -66,7 +66,8 @@ sql_error create table $tb (ts timestamp, col $i_binary ) sql_error create table $tb (ts timestamp, col $i_bigint ) sql_error create table $tb (ts timestamp, col $i_smallint ) sql_error create table $tb (ts timestamp, col $i_tinyint ) -sql_error create table $tb (ts timestamp, col $i_binary2 ) +sql create table $tb (ts timestamp, col $i_binary2 ) +sql drop table $tb sql_error create table $tb (ts timestamp, col $nchar ) sql create table $tb (ts timestamp, col nchar(20)) sql show tables @@ -105,7 +106,8 @@ sql_error create table $tb (ts timestamp, $ses int) sql_error create table $tb (ts timestamp, $int int) sql_error create table $tb (ts timestamp, $bint int) sql_error create table $tb (ts timestamp, $binary int) -sql_error create table $tb (ts timestamp, $str int) +sql create table $tb (ts timestamp, $str int) +sql drop table $tb sql_error create table $tb (ts timestamp, $tag int) sql_error create table $tb (ts timestamp, $tags int) sql_error create table $tb (ts timestamp, $sint int) @@ -157,7 +159,7 @@ print chinese_char_in_table_support test passed print ========== create_tb.sim case6: table_already_exists sql create table tbs (ts timestamp, col int) sql insert into tbs values (now, 1) -sql create table tbs (ts timestamp, col bool) +sql_error create table tbs (ts timestamp, col bool) #sql_error create table tb (ts timestamp, col bool) print table_already_exists test passed @@ -179,7 +181,7 @@ print table_already_exists test passed sql drop database $db sql show databases -if $rows != 0 then +if $rows != 2 then return -1 endi diff --git a/tests/script/tsim/parser/dbtbnameValidate.sim b/tests/script/tsim/parser/dbtbnameValidate.sim index 86ffbe5c37..939bc0ac4d 100644 --- a/tests/script/tsim/parser/dbtbnameValidate.sim +++ b/tests/script/tsim/parser/dbtbnameValidate.sim @@ -5,77 +5,72 @@ sql connect print ========== db name and table name check in create and drop, describe sql create database abc keep 36500 -sql create database 'abc123' -sql create database '_ab1234' -sql create database 'ABC123' -sql create database '_ABC123' +sql_error create database 'abc123' +sql_error create database '_ab1234' +sql_error create database 'ABC123' +sql_error create database '_ABC123' sql_error create database 'aABb123 ' sql_error create database ' xyz ' sql_error create database ' XYZ ' -sql use 'abc123' -sql use '_ab1234' -sql use 'ABC123' -sql use '_ABC123' +sql_error use 'abc123' +sql_error use '_ab1234' +sql_error use 'ABC123' +sql_error use '_ABC123' sql_error use 'aABb123' sql_error use ' xyz ' sql_error use ' XYZ ' -sql drop database 'abc123' -sql drop database '_ab1234' -sql_error drop database 'ABC123' -sql drop database '_ABC123' -sql_error drop database 'aABb123' -sql_error drop database ' xyz ' -sql_error drop database ' XYZ ' - +sql_error drop database if exists 'abc123' +sql_error drop database if exists '_ab1234' +sql_error drop database if exists 'ABC123' +sql_error drop database if exists '_ABC123' +sql_error drop database if exists 'aABb123' +sql_error drop database if exists ' xyz ' +sql_error drop database if exists ' XYZ ' sql use abc - sql create table abc.cc (ts timestamp, c int) -sql create table 'abc.Dd' (ts timestamp, c int) -sql create table 'abc'.ee (ts timestamp, c int) -sql create table 'abc'.'FF' (ts timestamp, c int) -sql create table abc.'gG' (ts timestamp, c int) - +sql_error create table 'abc.Dd' (ts timestamp, c int) +sql_error create table 'abc'.ee (ts timestamp, c int) +sql_error create table 'abc'.'FF' (ts timestamp, c int) +sql_error create table abc.'gG' (ts timestamp, c int) sql_error create table table.'a1' (ts timestamp, c int) sql_error create table 'table'.'b1' (ts timestamp, c int) sql_error create table 'table'.'b1' (ts timestamp, c int) - sql create table mt (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int, t2 nchar(20), t3 binary(20), t4 bigint, t5 smallint, t6 double) sql create table sub_001 using mt tags ( 1 , 'tag_nchar' , 'tag_bianry' , 4 , 5 , 6.1 ) sql_error create table sub_002 using mt tags( 2 , tag_nchar , tag_bianry , 4 , 5 , 6.2 ) sql insert into sub_dy_tbl using mt tags ( 3 , 'tag_nchar' , 'tag_bianry' , 4 , 5 , 6.3 ) values (now, 1, 2, 3.01, 4.02, 5, 6, true, 'binary_8', 'nchar_9') sql describe abc.cc -sql describe 'abc.Dd' -sql describe 'abc'.ee -sql describe 'abc'.'FF' -sql describe abc.'gG' +sql_error describe 'abc.Dd' +sql_error describe 'abc'.ee +sql_error describe 'abc'.'FF' +sql_error describe abc.'gG' sql describe cc -sql describe 'Dd' -sql describe ee -sql describe 'FF' -sql describe 'gG' +sql_error describe 'Dd' +sql_error describe ee +sql_error describe 'FF' +sql_error describe 'gG' sql describe mt sql describe sub_001 sql describe sub_dy_tbl -sql describe Dd -sql describe FF -sql describe gG +sql_error describe Dd +sql_error describe FF +sql_error describe gG sql drop table abc.cc -sql drop table 'abc.Dd' -sql drop table 'abc'.ee -sql drop table 'abc'.'FF' -sql drop table abc.'gG' +sql_error drop table 'abc.Dd' +sql_error drop table 'abc'.ee +sql_error drop table 'abc'.'FF' +sql_error drop table abc.'gG' sql drop table sub_001 - sql drop table sub_dy_tbl sql drop table mt diff --git a/tests/script/tsim/parser/distinct.sim b/tests/script/tsim/parser/distinct.sim index b90ca593ba..6d7dec0659 100644 --- a/tests/script/tsim/parser/distinct.sim +++ b/tests/script/tsim/parser/distinct.sim @@ -73,11 +73,10 @@ if $rows != 6 then return -1 endi - ### select distinct sql drop database $db sql show databases -if $rows != 0 then +if $rows != 2 then return -1 endi From d74f3fc20bc3df5152e183201aae885f0fa7621b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 21 Jul 2022 13:29:15 +0800 Subject: [PATCH 24/47] test: comment out unstable case --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index e9b7729535..336c6c0b7f 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -31,7 +31,7 @@ ./test.sh -f tsim/db/len.sim ./test.sh -f tsim/db/repeat.sim ./test.sh -f tsim/db/show_create_db.sim -./test.sh -f tsim/db/show_create_table.sim +# jira ./test.sh -f tsim/db/show_create_table.sim ./test.sh -f tsim/db/tables.sim ./test.sh -f tsim/db/taosdlog.sim From 9f3e7f021a64f60ab5fcff400d5a874e5ead25e0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 21 Jul 2022 13:29:51 +0800 Subject: [PATCH 25/47] test: comment out unstable case --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 336c6c0b7f..4d25d37f94 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -196,7 +196,7 @@ ./test.sh -f tsim/mnode/basic5.sim # ---- show -./test.sh -f tsim/show/basic.sim +# jira ./test.sh -f tsim/show/basic.sim # ---- table ./test.sh -f tsim/table/autocreate.sim From ac5119208471cc29f83d9ace35b7ab6990216850 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 21 Jul 2022 13:54:24 +0800 Subject: [PATCH 26/47] refactor(sync): add trace log --- source/libs/sync/src/syncAppendEntries.c | 12 ++--- source/libs/sync/test/sh/a.sh | 58 ++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 source/libs/sync/test/sh/a.sh diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 50c66172da..4295abeaa1 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -473,13 +473,13 @@ static bool syncNodeOnAppendEntriesBatchLogOK(SSyncNode* pSyncNode, SyncAppendEn SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); if (pMsg->prevLogIndex > myLastIndex) { - sDebug("vgId:%d sync log not ok, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); + sDebug("vgId:%d, sync log not ok, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); return false; } SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1); if (myPreLogTerm == SYNC_TERM_INVALID) { - sDebug("vgId:%d sync log not ok2, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); + sDebug("vgId:%d, sync log not ok2, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); return false; } @@ -487,7 +487,7 @@ static bool syncNodeOnAppendEntriesBatchLogOK(SSyncNode* pSyncNode, SyncAppendEn return true; } - sDebug("vgId:%d sync log not ok3, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); + sDebug("vgId:%d, sync log not ok3, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); return false; } @@ -500,13 +500,13 @@ static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); if (pMsg->prevLogIndex > myLastIndex) { - sDebug("vgId:%d sync log not ok, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); + sDebug("vgId:%d, sync log not ok, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); return false; } SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1); if (myPreLogTerm == SYNC_TERM_INVALID) { - sDebug("vgId:%d sync log not ok2, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); + sDebug("vgId:%d, sync log not ok2, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); return false; } @@ -514,7 +514,7 @@ static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries return true; } - sDebug("vgId:%d sync log not ok3, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); + sDebug("vgId:%d, sync log not ok3, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); return false; } diff --git a/source/libs/sync/test/sh/a.sh b/source/libs/sync/test/sh/a.sh new file mode 100644 index 0000000000..b2d8f88809 --- /dev/null +++ b/source/libs/sync/test/sh/a.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +if [ $# != 1 ] ; then + echo "Uasge: $0 log-path" + echo "" + exit 1 +fi + +logpath=$1 +echo "logpath: ${logpath}" + +echo "" +echo "clean old log ..." +rm -f ${logpath}/log.* + +echo "" +echo "generate log.dnode ..." +for dnode in `ls ${logpath} | grep dnode`;do + echo "generate log.${dnode}" + cat ${logpath}/${dnode}/log/taosdlog.* | grep SYN > ${logpath}/log.${dnode} +done + +echo "" +echo "generate vgId ..." +cat ${logpath}/log.dnode* | grep "vgId:" | grep -v ERROR | awk '{print $5}' | sort | uniq > ${logpath}/log.vgIds.tmp +echo "all vgIds:" > ${logpath}/log.vgIds +cat ${logpath}/log.dnode* | grep "vgId:" | grep -v ERROR | awk '{print $5}' | sort | uniq >> ${logpath}/log.vgIds +for dnode in `ls ${logpath} | grep dnode | grep -v log`;do + echo "" >> ${logpath}/log.vgIds + echo "" >> ${logpath}/log.vgIds + echo "${dnode}:" >> ${logpath}/log.vgIds + cat ${logpath}/${dnode}/log/taosdlog.* | grep SYN | grep "vgId:" | grep -v ERROR | awk '{print $5}' | sort | uniq >> ${logpath}/log.vgIds +done + +echo "" +echo "generate log.dnode.vgId ..." +for logdnode in `ls ${logpath}/log.dnode*`;do + for vgId in `cat ${logpath}/log.vgIds.tmp`;do + rowNum=`cat ${logdnode} | grep "${vgId}" | awk 'BEGIN{rowNum=0}{rowNum++}END{print rowNum}'` + #echo "-----${rowNum}" + if [ $rowNum -gt 0 ] ; then + echo "generate ${logdnode}.${vgId}" + cat ${logdnode} | grep "${vgId}" > ${logdnode}.${vgId} + fi + done +done + +echo "" +echo "generate log.dnode.main ..." +for file in `ls ${logpath}/log.dnode* | grep -v vgId`;do + echo "generate ${file}.main" + cat ${file} | awk '{ if(index($0, "sync open") > 0 || index($0, "sync close") > 0 || index($0, "become leader") > 0) {print $0} }' > ${file}.main +done + + + + +exit 0 From 3f49a274ba4c4b40c94b714297f8d3608aad1520 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 21 Jul 2022 13:57:48 +0800 Subject: [PATCH 27/47] fix:memory problems where using tag --- source/libs/parser/src/parInsert.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index c6b608ddb4..d6369157b0 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1018,7 +1018,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint end: for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { STagVal* p = (STagVal*)taosArrayGet(pTagVals, i); - if (IS_VAR_DATA_TYPE(p->type)) { + if (p->type == TSDB_DATA_TYPE_NCHAR) { taosMemoryFree(p->pData); } } From bab7abeacf51c4b448b957b7fc653a0d4a8104d3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 21 Jul 2022 14:01:05 +0800 Subject: [PATCH 28/47] test: restore 2.0 case --- tests/script/jenkins/basic.txt | 16 ++++++++-------- tests/script/tsim/parser/constCol.sim | 18 ++++-------------- tests/script/tsim/parser/import.sim | 7 ------- tests/script/tsim/parser/import_commit1.sim | 4 +--- tests/script/tsim/parser/import_commit2.sim | 4 +--- tests/script/tsim/parser/import_commit3.sim | 6 +----- tests/script/tsim/parser/insert_tb.sim | 2 -- tests/script/tsim/parser/tbnameIn_query.sim | 14 +------------- 8 files changed, 16 insertions(+), 55 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 4d25d37f94..8c5d522038 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -96,7 +96,7 @@ # jira ./test.sh -f tsim/parser/columnValue.sim ./test.sh -f tsim/parser/commit.sim # jira ./test.sh -f tsim/parser/condition.sim -## ./test.sh -f tsim/parser/constCol.sim +./test.sh -f tsim/parser/constCol.sim ./test.sh -f tsim/parser/create_db.sim ./test.sh -f tsim/parser/create_mt.sim # jira ./test.sh -f tsim/parser/create_tb_with_tag_name.sim @@ -113,11 +113,11 @@ # ./test.sh -f tsim/parser/groupby.sim # ./test.sh -f tsim/parser/having_child.sim ## ./test.sh -f tsim/parser/having.sim -## ./test.sh -f tsim/parser/import.sim -# ./test.sh -f tsim/parser/import_commit1.sim -# ./test.sh -f tsim/parser/import_commit2.sim -# ./test.sh -f tsim/parser/import_commit3.sim -## ./test.sh -f tsim/parser/import_file.sim +./test.sh -f tsim/parser/import_commit1.sim +./test.sh -f tsim/parser/import_commit2.sim +./test.sh -f tsim/parser/import_commit3.sim +# jira ./test.sh -f tsim/parser/import_file.sim +./test.sh -f tsim/parser/import.sim ## ./test.sh -f tsim/parser/insert_multiTbl.sim # ./test.sh -f tsim/parser/insert_tb.sim ## ./test.sh -f tsim/parser/interp.sim @@ -159,7 +159,7 @@ ./test.sh -f tsim/parser/stableOp.sim # ./test.sh -f tsim/parser/tags_dynamically_specifiy.sim # ./test.sh -f tsim/parser/tags_filter.sim -# jira ./test.sh -f tsim/parser/tbnameIn.sim +./test.sh -f tsim/parser/tbnameIn.sim ./test.sh -f tsim/parser/timestamp.sim ./test.sh -f tsim/parser/top_groupby.sim ./test.sh -f tsim/parser/topbot.sim @@ -328,7 +328,7 @@ ./test.sh -f tsim/vnode/stable_replica3_vnode3.sim # --- sync -./test.sh -f tsim/sync/3Replica1VgElect.sim +# jira ./test.sh -f tsim/sync/3Replica1VgElect.sim ./test.sh -f tsim/sync/3Replica5VgElect.sim ./test.sh -f tsim/sync/oneReplica1VgElect.sim ./test.sh -f tsim/sync/oneReplica5VgElect.sim diff --git a/tests/script/tsim/parser/constCol.sim b/tests/script/tsim/parser/constCol.sim index 5f50c950dd..5eb5b419fb 100644 --- a/tests/script/tsim/parser/constCol.sim +++ b/tests/script/tsim/parser/constCol.sim @@ -8,20 +8,16 @@ sql use db; sql create table t (ts timestamp, i int); sql create table st1 (ts timestamp, f1 int) tags(t1 int); sql create table st2 (ts timestamp, f2 int) tags(t2 int); - sql create table t1 using st1 tags(1); sql create table t2 using st2 tags(1); sql insert into t1 values(1575880055000, 1); sql insert into t1 values(1575880059000, 1); sql insert into t1 values(1575880069000, 1); - sql insert into t2 values(1575880055000, 2); sql select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts' 127.0.0.1:7111/restful/sql - print ==============select with user-defined columns sql select 'abc' as f, ts,f1 from t1 if $rows != 3 then @@ -301,13 +297,13 @@ if $data04 != 1.982700000 then endi print ======================udc with interval -sql select count(*), 'uuu' from t1 interval(1s) order by ts desc; +sql select count(*), 'uuu' from t1 interval(1s); if $rows != 3 then return -1 endi print ======================udc with tags -sql select t1,'abc',tbname from st1 +sql select distinct t1,'abc',tbname from st1 if $rows != 1 then return -1 endi @@ -343,31 +339,26 @@ if $rows != 0 then return -1 endi - print ======================udc with normal column group by - sql_error select from t1 sql_error select abc from t1 sql_error select abc as tu from t1 print ========================> td-1756 -sql_error select * from t1 where ts>now-1y -sql_error select * from t1 where ts>now-1n +sql select * from t1 where ts>now-1y +sql select * from t1 where ts>now-1n print ========================> td-1752 sql select * from db.st2 where t2 < 200 and t2 is not null; if $rows != 1 then return -1 endi - if $data00 != @19-12-09 16:27:35.000@ then return -1 endi - if $data01 != 2 then return -1 endi - if $data02 != 1 then return -1 endi @@ -376,7 +367,6 @@ sql select * from db.st2 where t2 > 200 or t2 is null; if $rows != 0 then return -1 endi - sql select * from st2 where t2 < 200 and t2 is null; if $rows != 0 then return -1 diff --git a/tests/script/tsim/parser/import.sim b/tests/script/tsim/parser/import.sim index 5946cff4e2..332ddba6b5 100644 --- a/tests/script/tsim/parser/import.sim +++ b/tests/script/tsim/parser/import.sim @@ -22,15 +22,10 @@ sql use $db sql create table tb (ts timestamp, c1 int, c2 timestamp) sql insert into tb values ('2019-05-05 11:30:00.000', 1, now) sql insert into tb values ('2019-05-05 12:00:00.000', 1, now) -sleep 500 sql import into tb values ('2019-05-05 11:00:00.000', -1, now) -sleep 500 sql import into tb values ('2019-05-05 11:59:00.000', -1, now) -sleep 500 sql import into tb values ('2019-05-04 08:00:00.000', -1, now) -sleep 500 sql import into tb values ('2019-05-04 07:59:00.000', -1, now) -sleep 500 sql select * from tb if $rows != 6 then @@ -57,11 +52,9 @@ endi print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 100 sql use $db sql select * from tb diff --git a/tests/script/tsim/parser/import_commit1.sim b/tests/script/tsim/parser/import_commit1.sim index 23259d8b01..e1aa0b6bb0 100644 --- a/tests/script/tsim/parser/import_commit1.sim +++ b/tests/script/tsim/parser/import_commit1.sim @@ -19,7 +19,7 @@ $stb = $stbPrefix . $i sql drop database $db -x step1 step1: -sql create database $db cache 16 +sql create database $db print ====== create tables sql use $db @@ -36,8 +36,6 @@ while $x < $rowNum endw print ====== tables created -sleep 500 - $ts = $ts0 + $delta $ts = $ts + 1 sql import into $tb values ( $ts , -1) diff --git a/tests/script/tsim/parser/import_commit2.sim b/tests/script/tsim/parser/import_commit2.sim index 49fca0d477..783a902818 100644 --- a/tests/script/tsim/parser/import_commit2.sim +++ b/tests/script/tsim/parser/import_commit2.sim @@ -18,7 +18,7 @@ $stb = $stbPrefix . $i sql drop database $db -x step1 step1: -sql create database $db cache 16 +sql create database $db print ====== create tables sql use $db @@ -35,8 +35,6 @@ while $x < $rowNum endw print ====== tables created -sleep 500 - $ts = $ts0 + $delta $ts = $ts + 1 sql import into $tb values ( $ts , -1) diff --git a/tests/script/tsim/parser/import_commit3.sim b/tests/script/tsim/parser/import_commit3.sim index d353c10387..1dc985cc1d 100644 --- a/tests/script/tsim/parser/import_commit3.sim +++ b/tests/script/tsim/parser/import_commit3.sim @@ -18,7 +18,7 @@ $stb = $stbPrefix . $i sql drop database $db -x step1 step1: -sql create database $db cache 16 +sql create database $db print ====== create tables sql use $db sql reset query cache @@ -35,16 +35,12 @@ while $x < $rowNum endw print ====== tables created -sleep 500 - $ts = $ts + 1 sql insert into $tb values ( $ts , -1, -1, -1, -1, -1) $ts = $ts0 + $delta $ts = $ts + 1 sql import into $tb values ( $ts , -2, -2, -2, -2, -2) -sleep 500 - sql show databases sql select count(*) from $tb diff --git a/tests/script/tsim/parser/insert_tb.sim b/tests/script/tsim/parser/insert_tb.sim index 4fa04e0625..134ed723f1 100644 --- a/tests/script/tsim/parser/insert_tb.sim +++ b/tests/script/tsim/parser/insert_tb.sim @@ -99,7 +99,6 @@ if $rows != 1 then endi sql drop database $db -sleep 100 sql create database $db sql use $db sql create table stb1 (ts timestamp, c1 int) tags(t1 int) @@ -132,7 +131,6 @@ if $data21 != 1.000000000 then endi sql drop database $db -sleep 100 sql create database $db sql use $db sql create table stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 nchar(10), c6 binary(20)) tags(t1 int, t2 bigint, t3 double, t4 float, t5 nchar(10)) diff --git a/tests/script/tsim/parser/tbnameIn_query.sim b/tests/script/tsim/parser/tbnameIn_query.sim index 33587cb7c1..6bc40fa028 100644 --- a/tests/script/tsim/parser/tbnameIn_query.sim +++ b/tests/script/tsim/parser/tbnameIn_query.sim @@ -104,7 +104,7 @@ endi ## tbname in can accpet Upper case table name print select count(*), t1 from $stb where tbname in ('ti_tb0', 'TI_tb1', 'TI_TB2') group by t1 order by t1 sql select count(*), t1 from $stb where tbname in ('ti_tb0', 'TI_tb1', 'TI_TB2') group by t1 order by t1 -if $rows != 3 then +if $rows != 1 then return -1 endi if $data00 != 10 then @@ -113,18 +113,6 @@ endi if $data01 != 0 then return -1 endi -if $data10 != 10 then - return -1 -endi -if $data11 != 1 then - return -1 -endi -if $data20 != 10 then - return -1 -endi -if $data21 != 2 then - return -1 -endi # multiple tbname in is not allowed NOW sql select count(*), t1 from $stb where tbname in ('ti_tb1', 'ti_tb300') and tbname in ('ti_tb5', 'ti_tb1000') group by t1 order by t1 asc From 3621aa55404d792a767a2026d0af1b6eea5b7a91 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 21 Jul 2022 14:06:38 +0800 Subject: [PATCH 29/47] fix: fix mac compile issue --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 24c7d5452a..11aac2114d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2162,7 +2162,7 @@ static SSDataBlock* sysTableScanUserSTables(SOperatorInfo* pOperator) { } pInfo->pRes->info.rows = 0; - pOperator->status == OP_EXEC_DONE; + pOperator->status = OP_EXEC_DONE; pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; From de8bb6c25ca279075989f3e498120720da2dba88 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 21 Jul 2022 14:40:16 +0800 Subject: [PATCH 30/47] fix: avoid rpc mem leak --- source/libs/transport/inc/transComm.h | 1 + source/libs/transport/src/transCli.c | 10 ++++++++-- source/libs/transport/src/transComm.c | 8 ++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 843798817d..9dd1a745d3 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -232,6 +232,7 @@ typedef struct { SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb); void transDestroyAsyncPool(SAsyncPool* pool); int transAsyncSend(SAsyncPool* pool, queue* mq); +bool transAsyncPoolIsEmpty(SAsyncPool* pool); #define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc) \ do { \ diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 07a698f883..f94a7f3c37 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -70,6 +70,8 @@ typedef struct SCliThrd { SCvtAddr cvtAddr; + SCliMsg* stopMsg; + bool quit; } SCliThrd; @@ -761,14 +763,17 @@ void cliConnCb(uv_connect_t* req, int status) { } static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd) { + if (!transAsyncPoolIsEmpty(pThrd->asyncPool)) { + pThrd->stopMsg = pMsg; + return; + } + pThrd->stopMsg = NULL; pThrd->quit = true; tDebug("cli work thread %p start to quit", pThrd); destroyCmsg(pMsg); destroyConnPool(pThrd->pool); uv_timer_stop(&pThrd->timer); uv_walk(pThrd->loop, cliWalkCb, NULL); - - // uv_stop(pThrd->loop); } static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { int64_t refId = (int64_t)(pMsg->msg.info.handle); @@ -925,6 +930,7 @@ static void cliAsyncCb(uv_async_t* handle) { if (count >= 2) { tTrace("cli process batch size:%d", count); } + if (pThrd->stopMsg != NULL) cliHandleQuit(pThrd->stopMsg, pThrd); } static void* cliWorkThread(void* arg) { diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index c89bbd408b..c3cba3118c 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -228,6 +228,14 @@ int transAsyncSend(SAsyncPool* pool, queue* q) { } return uv_async_send(async); } +bool transAsyncPoolIsEmpty(SAsyncPool* pool) { + for (int i = 0; i < pool->nAsync; i++) { + uv_async_t* async = &(pool->asyncs[i]); + SAsyncItem* item = async->data; + if (!QUEUE_IS_EMPTY(&item->qmsg)) return false; + } + return true; +} void transCtxInit(STransCtx* ctx) { // init transCtx From 8915486ec148b9c05c23c7b230b17bcf0d5fb3bd Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 21 Jul 2022 14:51:02 +0800 Subject: [PATCH 31/47] fix(sync): raft store persist --- source/libs/sync/src/syncRaftStore.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index f552570337..fbfeb031f6 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -108,10 +108,10 @@ int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { cJSON *pRoot = cJSON_CreateObject(); char u64Buf[128] = {0}; - snprintf(u64Buf, sizeof(u64Buf), "" PRIu64 "", pRaftStore->currentTerm); + snprintf(u64Buf, sizeof(u64Buf), "%" PRIu64 "", pRaftStore->currentTerm); cJSON_AddStringToObject(pRoot, "current_term", u64Buf); - snprintf(u64Buf, sizeof(u64Buf), "" PRIu64 "", pRaftStore->voteFor.addr); + snprintf(u64Buf, sizeof(u64Buf), "%" PRIu64 "", pRaftStore->voteFor.addr); cJSON_AddStringToObject(pRoot, "vote_for_addr", u64Buf); cJSON_AddNumberToObject(pRoot, "vote_for_vgid", pRaftStore->voteFor.vgId); @@ -142,11 +142,11 @@ int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { cJSON *pCurrentTerm = cJSON_GetObjectItem(pRoot, "current_term"); ASSERT(cJSON_IsString(pCurrentTerm)); - sscanf(pCurrentTerm->valuestring, "" PRIu64 "", &(pRaftStore->currentTerm)); + sscanf(pCurrentTerm->valuestring, "%" PRIu64 "", &(pRaftStore->currentTerm)); cJSON *pVoteForAddr = cJSON_GetObjectItem(pRoot, "vote_for_addr"); ASSERT(cJSON_IsString(pVoteForAddr)); - sscanf(pVoteForAddr->valuestring, "" PRIu64 "", &(pRaftStore->voteFor.addr)); + sscanf(pVoteForAddr->valuestring, "%" PRIu64 "", &(pRaftStore->voteFor.addr)); cJSON *pVoteForVgid = cJSON_GetObjectItem(pRoot, "vote_for_vgid"); pRaftStore->voteFor.vgId = pVoteForVgid->valueint; @@ -188,11 +188,11 @@ cJSON *raftStore2Json(SRaftStore *pRaftStore) { cJSON *pRoot = cJSON_CreateObject(); if (pRaftStore != NULL) { - snprintf(u64buf, sizeof(u64buf), "" PRIu64 "", pRaftStore->currentTerm); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pRaftStore->currentTerm); cJSON_AddStringToObject(pRoot, "currentTerm", u64buf); cJSON *pVoteFor = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "" PRIu64 "", pRaftStore->voteFor.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pRaftStore->voteFor.addr); cJSON_AddStringToObject(pVoteFor, "addr", u64buf); { uint64_t u64 = pRaftStore->voteFor.addr; From f5a326dbf44e0350dc2ff9f8701a9e18cef2bc3e Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 21 Jul 2022 15:16:57 +0800 Subject: [PATCH 32/47] feat(stream): optimize update data check --- include/libs/stream/tstreamUpdate.h | 3 ++- source/libs/stream/src/streamUpdate.c | 20 ++++++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index 21a1515d8f..6e4a8d62d0 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -33,11 +33,12 @@ typedef struct SUpdateInfo { int64_t watermark; TSKEY minTS; SScalableBf* pCloseWinSBF; + SHashObj* pMap; } SUpdateInfo; SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark); SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark); -bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts); +bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts); void updateInfoDestroy(SUpdateInfo *pInfo); void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo); void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo); diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index d0fb9c22e1..b7b635e28f 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -15,9 +15,12 @@ #include "tstreamUpdate.h" #include "ttime.h" +#include "query.h" #define DEFAULT_FALSE_POSITIVE 0.01 -#define DEFAULT_BUCKET_SIZE 131072 +#define DEFAULT_BUCKET_SIZE 1310720 +#define DEFAULT_MAP_CAPACITY 1310720 +#define DEFAULT_MAP_SIZE (DEFAULT_MAP_CAPACITY * 10) #define ROWS_PER_MILLISECOND 1 #define MAX_NUM_SCALABLE_BF 100000 #define MIN_NUM_SCALABLE_BF 10 @@ -120,6 +123,8 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma } pInfo->numBuckets = DEFAULT_BUCKET_SIZE; pInfo->pCloseWinSBF = NULL; + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pMap = taosHashInit(DEFAULT_MAP_CAPACITY, hashFn, true, HASH_NO_LOCK); return pInfo; } @@ -149,8 +154,9 @@ static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { return res; } -bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts) { +bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { int32_t res = TSDB_CODE_FAILED; + TSKEY* pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); uint64_t index = ((uint64_t)tableId) % pInfo->numBuckets; TSKEY maxTs = *(TSKEY *)taosArrayGet(pInfo->pTsBuckets, index); if (ts < maxTs - pInfo->watermark) { @@ -167,7 +173,13 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts) { res = tScalableBfPut(pSBf, &ts, sizeof(TSKEY)); } - if (maxTs < ts) { + int32_t size = taosHashGetSize(pInfo->pMap); + if ( (!pMapMaxTs && size < DEFAULT_MAP_SIZE) || (pMapMaxTs && *pMapMaxTs < ts)) { + taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), &ts, sizeof(TSKEY)); + return false; + } + + if ( !pMapMaxTs && maxTs < ts ) { taosArraySet(pInfo->pTsBuckets, index, &ts); return false; } @@ -177,7 +189,7 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts) { } else if (res == TSDB_CODE_SUCCESS) { return false; } - + qDebug("===stream===bucket:%d, tableId:%" PRIu64 ", maxTs:" PRIu64 ", maxMapTs:" PRIu64 ", ts:%" PRIu64, index, tableId, maxTs, *pMapMaxTs, ts); // check from tsdb api return true; } From fc26ff4310f243cdfeffc55e2071c582ca9b33a3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 21 Jul 2022 15:21:09 +0800 Subject: [PATCH 33/47] test: restore 2.0 case --- tests/script/jenkins/basic.txt | 10 +- tests/script/tsim/parser/insert_multiTbl.sim | 10 +- tests/script/tsim/parser/insert_tb.sim | 5 +- tests/script/tsim/parser/interp.sim | 1 - tests/script/tsim/parser/interp_test.sim | 2 +- tests/script/tsim/parser/last_cache.sim | 2 - tests/script/tsim/parser/last_cache_query.sim | 47 ++---- tests/script/tsim/parser/lastrow.sim | 2 - tests/script/tsim/parser/lastrow_query.sim | 153 ++++++++++-------- tests/system-test/fulltest.sh | 2 +- 10 files changed, 105 insertions(+), 129 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 8c5d522038..f319de4c2f 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -118,16 +118,16 @@ ./test.sh -f tsim/parser/import_commit3.sim # jira ./test.sh -f tsim/parser/import_file.sim ./test.sh -f tsim/parser/import.sim -## ./test.sh -f tsim/parser/insert_multiTbl.sim -# ./test.sh -f tsim/parser/insert_tb.sim -## ./test.sh -f tsim/parser/interp.sim +./test.sh -f tsim/parser/insert_multiTbl.sim +./test.sh -f tsim/parser/insert_tb.sim +# jira ./test.sh -f tsim/parser/interp.sim # ./test.sh -f tsim/parser/join.sim # ./test.sh -f tsim/parser/join_manyblocks.sim ## ./test.sh -f tsim/parser/join_multitables.sim # ./test.sh -f tsim/parser/join_multivnode.sim -# ./test.sh -f tsim/parser/last_cache.sim +./test.sh -f tsim/parser/last_cache.sim ## ./test.sh -f tsim/parser/last_groupby.sim -# ./test.sh -f tsim/parser/lastrow.sim +# jira ./test.sh -f tsim/parser/lastrow.sim ## ./test.sh -f tsim/parser/like.sim # ./test.sh -f tsim/parser/limit.sim # ./test.sh -f tsim/parser/limit1.sim diff --git a/tests/script/tsim/parser/insert_multiTbl.sim b/tests/script/tsim/parser/insert_multiTbl.sim index 85c58ef3d3..78d3352378 100644 --- a/tests/script/tsim/parser/insert_multiTbl.sim +++ b/tests/script/tsim/parser/insert_multiTbl.sim @@ -11,9 +11,9 @@ sql create table mul_st (ts timestamp, col1 int) tags (tag1 int) # case: insert multiple recordes for multiple table in a query print =========== insert_multiTbl.sim case: insert multiple records for multiple table in a query -$ts = 1500000000000 +$ts = 1600000000000 sql insert into mul_t0 using mul_st tags(0) values ( $ts , 0) ( $ts + 1s, 1) ( $ts + 2s, 2) mul_t1 using mul_st tags(1) values ( $ts , 10) ( $ts + 1s, 11) ( $ts + 2s, 12) mul_t2 using mul_st tags(2) values ( $ts , 20) ( $ts + 1s, 21) ( $ts + 2s, 22) mul_t3 using mul_st tags(3) values ( $ts , 30) ( $ts + 1s, 31) ( $ts + 2s, 32) -sql select * from mul_st +sql select * from mul_st order by ts, col1 ; print rows = $rows if $rows != 12 then return -1 @@ -40,10 +40,10 @@ endi # insert values for specified columns sql create table mul_st1 (ts timestamp, col1 int, col2 float, col3 binary(10)) tags (tag1 int, tag2 int, tag3 binary(8)) print =========== insert values for specified columns for multiple table in a query -$ts = 1500000000000 +$ts = 1600000000000 sql insert into mul_t10 (ts, col1, col3) using mul_st1 (tag1, tag3) tags(0, 'tag3-0') values ( $ts , 00, 'binary00') ( $ts + 1s, 01, 'binary01') ( $ts + 2s, 02, 'binary02') mul_t11 (ts, col1, col3) using mul_st1 (tag1, tag3) tags(1, 'tag3-0') values ( $ts , 10, 'binary10') ( $ts + 1s, 11, 'binary11') ( $ts + 2s, 12, 'binary12') mul_t12 (ts, col1, col3) using mul_st1 (tag1, tag3) tags(2, 'tag3-0') values ( $ts , 20, 'binary20') ( $ts + 1s, 21, 'binary21') ( $ts + 2s, 22, 'binary22') mul_t13 (ts, col1, col3) using mul_st1 (tag1, tag3) tags(3, 'tag3-0') values ( $ts , 30, 'binary30') ( $ts + 1s, 31, 'binary31') ( $ts + 2s, 32, 'binary32') -sql select * from mul_st1 +sql select * from mul_st1 order by ts, col1 ; print rows = $rows if $rows != 12 then return -1 @@ -58,7 +58,7 @@ endi if $data92 != NULL then return -1 endi -if $data93 != @binary30@ then +if $data93 != @binary12@ then return -1 endi diff --git a/tests/script/tsim/parser/insert_tb.sim b/tests/script/tsim/parser/insert_tb.sim index 134ed723f1..426ac4001f 100644 --- a/tests/script/tsim/parser/insert_tb.sim +++ b/tests/script/tsim/parser/insert_tb.sim @@ -53,7 +53,7 @@ endi $col1 = 2 $col3 = 3 $col5 = 5 -sql create table $tb using $mt tags( $tag1 ) +sql create table if not exists $tb using $mt tags( $tag1 ) sql insert into $tb ( ts, col1, col3, col5) values ( $ts + 2000a, $col1 , $col3 , $col5 ) sql select * from $tb order by ts desc if $rows != 3 then @@ -144,7 +144,7 @@ sql insert into tb1 values ('2018-09-17 09:00:00.000', '1', 1, 1, 1, '涛思ncha sql insert into tb2 values ('2018-09-17 09:00:00.000', 1, '1', 1, 1, '涛思nchar', 'quoted bigint') sql insert into tb3 values ('2018-09-17 09:00:00.000', 1, 1, '1', 1, '涛思nchar', 'quoted float') sql insert into tb4 values ('2018-09-17 09:00:00.000', 1, 1, 1, '1', '涛思nchar', 'quoted double') -sql select * from stb +sql select * from stb order by t1 if $rows != 5 then return -1 endi @@ -226,5 +226,4 @@ endi # return -1 #endi - system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/interp.sim b/tests/script/tsim/parser/interp.sim index 4bb273af46..cd67083701 100644 --- a/tests/script/tsim/parser/interp.sim +++ b/tests/script/tsim/parser/interp.sim @@ -59,7 +59,6 @@ run tsim/parser/interp_test.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/tsim/parser/interp_test.sim b/tests/script/tsim/parser/interp_test.sim index 8eac8a41d3..0bdf97a957 100644 --- a/tests/script/tsim/parser/interp_test.sim +++ b/tests/script/tsim/parser/interp_test.sim @@ -21,7 +21,7 @@ print ====== use db sql use $db ##### select interp from table -print ====== select intp from table +print ====== select interp from table $tb = $tbPrefix . 0 ## interp(*) from tb sql select interp(*) from $tb where ts = $ts0 diff --git a/tests/script/tsim/parser/last_cache.sim b/tests/script/tsim/parser/last_cache.sim index 7ffb3749aa..40c6e4ce12 100644 --- a/tests/script/tsim/parser/last_cache.sim +++ b/tests/script/tsim/parser/last_cache.sim @@ -4,14 +4,12 @@ system sh/exec.sh -n dnode1 -s start sql connect print ======================== dnode1 start - $db = testdb sql drop database if exists $db sql create database $db cachemodel 'last_value' sql use $db sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int) - sql create table tb1 using st2 tags (1); sql create table tb2 using st2 tags (2); sql create table tb3 using st2 tags (3); diff --git a/tests/script/tsim/parser/last_cache_query.sim b/tests/script/tsim/parser/last_cache_query.sim index 2acd000585..7bafe82f5d 100644 --- a/tests/script/tsim/parser/last_cache_query.sim +++ b/tests/script/tsim/parser/last_cache_query.sim @@ -1,11 +1,8 @@ -sleep 100 sql connect $db = testdb - sql use $db - print "test tb1" sql select last(ts) from tb1 @@ -17,7 +14,6 @@ if $data00 != @21-05-12 10:10:12.000@ then return -1 endi - sql select last(f1) from tb1 if $rows != 1 then return -1 @@ -49,7 +45,6 @@ if $data04 != @70-01-01 07:59:57.000@ then return -1 endi - sql select last(tb1.*,ts,f4) from tb1 if $rows != 1 then return -1 @@ -79,11 +74,7 @@ if $data06 != @70-01-01 07:59:57.000@ then return -1 endi - - - print "test tb2" - sql select last(ts) from tb2 if $rows != 1 then return -1 @@ -93,7 +84,6 @@ if $data00 != @21-05-11 10:11:15.000@ then return -1 endi - sql select last(f1) from tb2 if $rows != 1 then return -1 @@ -127,7 +117,6 @@ if $data04 != @70-01-01 07:59:56.999@ then endi endi - sql select last(tb2.*,ts,f4) from tb2 if $rows != 1 then return -1 @@ -161,12 +150,6 @@ if $data06 != @70-01-01 07:59:56.999@ then endi endi - - - - - - print "test tbd" sql select last(*) from tbd if $rows != 1 then @@ -190,18 +173,12 @@ if $data04 != NULL then return -1 endi - - print "test tbe" sql select last(*) from tbe if $rows != 0 then return -1 endi - - - - print "test stable" sql select last(ts) from st2 if $rows != 1 then @@ -212,7 +189,6 @@ if $data00 != @21-05-12 10:10:12.000@ then return -1 endi - sql select last(f1) from st2 if $rows != 1 then return -1 @@ -274,8 +250,13 @@ if $data06 != @70-01-01 07:59:57.000@ then return -1 endi +sql select last(*), id from st2 group by id order by id +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49 -sql select last(*) from st2 group by id if $rows != 5 then return -1 endi @@ -311,12 +292,9 @@ endi if $data13 != -8 then return -1 endi -if $data14 != @70-01-01 07:59:57.996@ then if $data14 != @70-01-01 07:59:58.-04@ then - print $data14 return -1 endi -endi if $data15 != 2 then return -1 endi @@ -326,18 +304,16 @@ endi if $data21 != 24 then return -1 endi -if $data22 != 8.000000000 then +if $data22 != 11.000000000 then print $data02 return -1 endi if $data23 != 25 then return -1 endi -if $data24 != @70-01-01 07:59:56.996@ then -if $data24 != @70-01-01 07:59:57.-04@ then +if $data24 != @70-01-01 07:59:57.-04@ then = return -1 endi -endi if $data25 != 3 then return -1 endi @@ -354,11 +330,9 @@ endi if $data33 != 27 then return -1 endi -if $data34 != @70-01-01 07:59:55.996@ then if $data34 != @70-01-01 07:59:56.-04@ then return -1 endi -endi if $data35 != 4 then return -1 endi @@ -375,18 +349,15 @@ endi if $data43 != 35 then return -1 endi -if $data44 != @70-01-01 07:59:55.995@ then if $data44 != @70-01-01 07:59:56.-05@ then return -1 endi -endi if $data45 != 5 then return -1 endi - print "test tbn" -sql create table tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) +sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) sql insert into tbn values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000) sql insert into tbn values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000) sql insert into tbn values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000) diff --git a/tests/script/tsim/parser/lastrow.sim b/tests/script/tsim/parser/lastrow.sim index d6638f2e98..db92e87de0 100644 --- a/tests/script/tsim/parser/lastrow.sim +++ b/tests/script/tsim/parser/lastrow.sim @@ -58,11 +58,9 @@ run tsim/parser/lastrow_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 100 run tsim/parser/lastrow_query.sim diff --git a/tests/script/tsim/parser/lastrow_query.sim b/tests/script/tsim/parser/lastrow_query.sim index 3fd88cfc1b..cb523d5c8e 100644 --- a/tests/script/tsim/parser/lastrow_query.sim +++ b/tests/script/tsim/parser/lastrow_query.sim @@ -1,4 +1,3 @@ -sleep 100 sql connect $dbPrefix = lr_db @@ -17,7 +16,7 @@ $stb = $stbPrefix . $i sql use $db print ========>TD-3231 last_row with group by column error -sql_error select last_row(c1) from $stb group by c1; +sql select last_row(c1) from $stb group by c1; ##### select lastrow from STable with two vnodes, timestamp decreases from tables in vnode0 to tables in vnode1 sql select last_row(*) from $stb @@ -67,91 +66,111 @@ if $row != 21600 then endi #regression test case 3 -sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 limit 1 +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by t1 interval(1h) fill(NULL) limit 1 if $row != 2 then return -1 endi +#if $data01 != 7 then +# return -1 +#endi +#if $data02 != 7 then +# return -1 +#endi +#if $data03 != 59 then +# print expect 59, actual: $data03 +# return -1 +#endi +#if $data04 != 7 then +# return -1 +#endi +#if $data11 != 8 then +# return -1 +#endi +#if $data12 != 8 then +# return -1 +#endi +#if $data13 != NULL then +# return -1 +#endi -if $data01 != 7 then - return -1 -endi - -if $data02 != 7 then - return -1 -endi - -if $data03 != 59 then - print expect 59, actual: $data03 - return -1 -endi - -if $data04 != 7 then - return -1 -endi - -if $data11 != 8 then - return -1 -endi - -if $data12 != 8 then - return -1 -endi - -if $data13 != NULL then - return -1 -endi - -sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 limit 9 +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by t1 interval(1h) fill(NULL) limit 9 if $rows != 18 then return -1 endi -sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 limit 12 +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by t1 interval(1h) fill(NULL) limit 12 if $rows != 24 then return -1 endi -sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 limit 25 +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by t1 interval(1h) fill(NULL) limit 25 if $rows != 48 then return -1 endi -sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 limit 25 offset 1 +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by t1 interval(1h) fill(NULL) limit 25 offset 1 if $rows != 46 then return -1 endi -sql select t1,t1,count(*),tbname,t1,t1,tbname from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1s) fill(NULL) group by tbname, t1 slimit 2 soffset 0 limit 250000 offset 1 -if $rows != 172798 then - return -1 -endi - -sql select t1,t1,count(*),tbname,t1,t1,tbname from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1s) fill(NULL) group by tbname, t1 slimit 1 soffset 1 limit 250000 offset 1 -if $rows != 86399 then - return -1 -endi - -sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts DESC limit 30 -if $rows != 48 then - return -1 -endi - -sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts DESC limit 2 -if $rows != 4 then - return -1 -endi - -sql select t1,t1,count(*),tbname,t1,t1,tbname from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1s) fill(NULL) group by tbname, t1 order by ts desc slimit 1 soffset 1 limit 250000 offset 1 -if $rows != 86399 then - return -1 -endi - -sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts desc limit 1 +sql select count(*),tbname,t1,t1,tbname from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by tbname, t1 interval(1d) fill(NULL) slimit 2 if $rows != 2 then return -1 endi -sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts desc limit 25 offset 1 +sql select count(*),tbname,t1,t1,tbname from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by tbname, t1 interval(1d) fill(NULL) slimit 2 soffset 1 +if $rows != 1 then + return -1 +endi + +sql select count(*),tbname,t1,t1,tbname from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by tbname, t1 interval(1d) fill(NULL) slimit 1 +if $rows != 1 then + return -1 +endi + +sql select count(*),tbname,t1,t1,tbname from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by tbname, t1 interval(1d) fill(NULL) slimit 1 soffset 1 +if $rows != 0 then + return -1 +endi + +sql select count(*),tbname,t1,t1,tbname from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by tbname, t1 interval(1d) fill(NULL) slimit 1 soffset 0 +if $rows != 0 then + return -1 +endi + +return + +sql select t1,t1,count(*),tbname,t1,t1,tbname from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by tbname, t1 interval(1s) fill(NULL) slimit 2 soffset 0 limit 250000 offset 1 +if $rows != 172799 then + return -1 +endi + +sql select t1,t1,count(*),tbname,t1,t1,tbname from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by tbname, t1 interval(1s) fill(NULL) slimit 1 soffset 0 limit 250000 offset 1 +if $rows != 86399 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by t1 interval(1h) fill(NULL) order by ts DESC limit 30 +if $rows != 30 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by t1 interval(1h) fill(NULL) limit 2 +if $rows != 2 then + return -1 +endi + +sql select t1,t1,count(*),tbname,t1,t1,tbname from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by tbname, t1 interval(1s) fill(NULL) slimit 1 soffset 1 limit 250000 offset 1 +if $rows != 86399 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by t1 interval(1h) fill(NULL) order by ts desc limit 1 +if $rows != 2 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by t1 interval(1h) fill(NULL) limit 25 offset 1 if $rows != 46 then return -1 endi @@ -166,12 +185,10 @@ sql select last_row(*) from t1 if $rows != 0 then return -1 endi - sql select last_row(*) from m1 if $rows != 0 then return -1 endi - sql select last_row(*) from m1 where tbname in ('t1') if $rows != 0 then return -1 @@ -189,16 +206,13 @@ sql select last_row(ts), 'abc', 1234.9384, ts from t1 if $rows != 1 then return -1 endi - if $data01 != @abc@ then print expect abc, actual $data02 return -1 endi - if $data02 != 1234.938400000 then return -1 endi - if $data03 != @19-01-01 01:01:01.000@ then print expect 19-01-01 01:01:01.000, actual:$data03 return -1 @@ -209,15 +223,12 @@ sql select last_row(*), ts, 'abc', 123.981, tbname from m1 if $rows != 1 then return -1 endi - if $data02 != @19-01-01 01:01:01.000@ then return -1 endi - if $data03 != @abc@ then return -1 endi - if $data04 != 123.981000000 then print expect 123.981000000, actual: $data04 return -1 diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 4588474753..978fa136e6 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -212,7 +212,7 @@ python3 ./test.py -f 7-tmq/tmqUdf.py # python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py # python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py -python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py +# python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py #------------querPolicy 2----------- From 6c683902ff111c9335cc9c45eaa736895a5ef612 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 21 Jul 2022 15:23:18 +0800 Subject: [PATCH 34/47] fix: fix show create table issue --- source/libs/parser/src/parUtil.c | 6 ++++-- tests/script/api/batchprepare.c | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index f98b195039..74d5f03dc1 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -866,13 +866,15 @@ STableCfg* tableCfgDup(STableCfg* pCfg) { memcpy(pNew, pCfg, sizeof(*pNew)); if (NULL != pNew->pComment) { - pNew->pComment = strdup(pNew->pComment); + pNew->pComment = taosMemoryCalloc(pNew->commentLen + 1, 1); + memcpy(pNew->pComment, pCfg->pComment, pNew->commentLen); } if (NULL != pNew->pFuncs) { pNew->pFuncs = taosArrayDup(pNew->pFuncs); } if (NULL != pNew->pTags) { - pNew->pTags = strdup(pNew->pTags); + pNew->pTags = taosMemoryCalloc(pNew->tagsLen + 1, 1); + memcpy(pNew->pTags, pCfg->pTags, pNew->tagsLen); } int32_t schemaSize = (pCfg->numOfColumns + pCfg->numOfTags) * sizeof(SSchema); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index e1aa1991a4..ada2039460 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -250,7 +250,7 @@ CaseCtrl gCaseCtrl = { #endif -#if 0 +#if 1 CaseCtrl gCaseCtrl = { // default .precision = TIME_PRECISION_MILLI, .bindNullNum = 0, @@ -282,7 +282,7 @@ CaseCtrl gCaseCtrl = { // default }; #endif -#if 1 +#if 0 CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindNullNum = 1, .printCreateTblSql = false, From c4bb5a1a527cc319aaf794e4a35a7c728d9e8340 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 21 Jul 2022 15:25:46 +0800 Subject: [PATCH 35/47] refactor(sync): add log tools --- source/libs/sync/test/sh/a.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/test/sh/a.sh b/source/libs/sync/test/sh/a.sh index b2d8f88809..4b1f1d0d5b 100644 --- a/source/libs/sync/test/sh/a.sh +++ b/source/libs/sync/test/sh/a.sh @@ -52,7 +52,9 @@ for file in `ls ${logpath}/log.dnode* | grep -v vgId`;do cat ${file} | awk '{ if(index($0, "sync open") > 0 || index($0, "sync close") > 0 || index($0, "become leader") > 0) {print $0} }' > ${file}.main done - +echo "" +echo "generate log.leader.term ..." +cat ${logpath}/*.main | grep "become leader" | grep -v "config change" | awk '{print $5,$0}' | awk -F, '{print $4"_"$0}' | sort -k1 > ${logpath}/log.leader.term exit 0 From ae891ab0ba989e979b6412c69b6408933da71b3e Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 21 Jul 2022 15:34:23 +0800 Subject: [PATCH 36/47] test: add test case for tmq --- tests/system-test/7-tmq/tmqDropStbCtb.py | 289 +++++++++++++++++++++++ 1 file changed, 289 insertions(+) create mode 100644 tests/system-test/7-tmq/tmqDropStbCtb.py diff --git a/tests/system-test/7-tmq/tmqDropStbCtb.py b/tests/system-test/7-tmq/tmqDropStbCtb.py new file mode 100644 index 0000000000..d9e675ddc6 --- /dev/null +++ b/tests/system-test/7-tmq/tmqDropStbCtb.py @@ -0,0 +1,289 @@ + +import taos +import sys +import time +import socket +import os +import threading +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.snapshot = 0 + self.vgroups = 4 + self.ctbNum = 100 + self.rowsPerTbl = 1000 + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def waitSubscriptionExit(self, max_wait_count=20): + wait_cnt = 0 + while (wait_cnt < max_wait_count): + tdSql.query("show subscriptions") + if tdSql.getRows() == 0: + break + else: + time.sleep(2) + wait_cnt += 1 + + tdLog.info("wait subscriptions exit for %d s"%wait_cnt) + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 1000, + 'batchNum': 1000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx", + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + # drop some ctbs + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 1000, + 'batchNum': 1000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'endTs': 0, + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + paraDict['snapshot'] = self.snapshot + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + # again create one new stb1 + paraDict["stbName"] = 'stb1' + paraDict['ctbPrefix'] = 'ctb1n_' + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("async insert data") + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tdLog.info("create topics from database") + topicFromDb = 'topic_dbt' + tdSql.execute("create topic %s as database %s" %(topicFromDb, paraDict['dbName'])) + + if self.snapshot == 0: + consumerId = 0 + elif self.snapshot == 1: + consumerId = 1 + + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) + topicList = topicFromDb + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tdLog.info("drop some ctables") + paraDict["stbName"] = 'stb' + paraDict['ctbPrefix'] = 'ctb' + paraDict["ctbStartIdx"] = paraDict["ctbStartIdx"] + int(paraDict["ctbNum"] * 3 / 4) # drop 1/4 ctbls + paraDict["ctbNum"] = int(paraDict["ctbNum"] / 4) + # tdSql.execute("drop table %s.%s" %(paraDict['dbName'], paraDict['stbName'])) + tmqCom.drop_ctable(tdSql, dbname=paraDict['dbName'], count=paraDict["ctbNum"], default_ctbname_prefix=paraDict["ctbPrefix"], ctbStartIdx=paraDict["ctbStartIdx"]) + + pInsertThread.join() + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + + if not ((totalConsumeRows > expectrowcnt / 2) and (totalConsumeRows < expectrowcnt)): + tdLog.exit("tmq consume rows error with snapshot = 0!") + + tdLog.info("wait subscriptions exit ....") + self.waitSubscriptionExit() + + tdSql.query("drop topic %s"%topicFromDb) + tdLog.info("success dorp topic: %s"%topicFromDb) + tdLog.printNoPrefix("======== test case 1 end ...... ") + + # drop one stb + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 1000, + 'batchNum': 1000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'endTs': 0, + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + paraDict['snapshot'] = self.snapshot + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + # again create one new stb1 + paraDict["stbName"] = 'stb2' + paraDict['ctbPrefix'] = 'ctb2n_' + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("async insert data") + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tdLog.info("create topics from database") + topicFromDb = 'topic_dbt' + tdSql.execute("create topic %s as database %s" %(topicFromDb, paraDict['dbName'])) + + if self.snapshot == 0: + consumerId = 0 + elif self.snapshot == 1: + consumerId = 1 + + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) + topicList = topicFromDb + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tdLog.info("drop one stable") + paraDict["stbName"] = 'stb1' + tdSql.execute("drop table %s.%s" %(paraDict['dbName'], paraDict['stbName'])) + + pInsertThread.join() + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + + if not ((totalConsumeRows > expectrowcnt / 2) and (totalConsumeRows < expectrowcnt)): + tdLog.exit("tmq consume rows error with snapshot = 0!") + + tdLog.info("wait subscriptions exit ....") + self.waitSubscriptionExit() + + tdSql.query("drop topic %s"%topicFromDb) + tdLog.info("success dorp topic: %s"%topicFromDb) + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def run(self): + tdLog.printNoPrefix("=============================================") + tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") + self.snapshot = 0 + self.prepareTestEnv() + self.tmqCase1() + self.tmqCase2() + + tdLog.printNoPrefix("====================================================================") + tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") + self.snapshot = 1 + self.prepareTestEnv() + self.tmqCase1() + self.tmqCase2() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 6f5479b02fddbb0ca54be69a95d8a40eea7f2090 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Thu, 21 Jul 2022 15:38:59 +0800 Subject: [PATCH 37/47] doc: reconstruct get-started, deployment and tdinternals --- .../01-docker.md} | 0 docs/zh/05-get-started/03-package.md | 321 +++++++++++++ docs/zh/05-get-started/06-first-use.md | 134 ++++++ docs/zh/10-cluster/02-cluster-mgmt.md | 105 ---- docs/zh/10-cluster/_category_.yml | 1 - .../01-deploy.md | 76 ++- docs/zh/10-deployment/03-k8s.md | 451 ++++++++++++++++++ docs/zh/10-deployment/05-helm.md | 415 ++++++++++++++++ docs/zh/10-deployment/_category_.yml | 1 + .../zh/{10-cluster => 10-deployment}/index.md | 4 +- .../03-high-availability.md | 0 .../05-load-balance.md} | 0 12 files changed, 1398 insertions(+), 110 deletions(-) rename docs/zh/{27-train-faq/03-docker.md => 05-get-started/01-docker.md} (100%) create mode 100644 docs/zh/05-get-started/03-package.md create mode 100644 docs/zh/05-get-started/06-first-use.md delete mode 100644 docs/zh/10-cluster/02-cluster-mgmt.md delete mode 100644 docs/zh/10-cluster/_category_.yml rename docs/zh/{10-cluster => 10-deployment}/01-deploy.md (64%) create mode 100644 docs/zh/10-deployment/03-k8s.md create mode 100644 docs/zh/10-deployment/05-helm.md create mode 100644 docs/zh/10-deployment/_category_.yml rename docs/zh/{10-cluster => 10-deployment}/index.md (82%) rename docs/zh/{10-cluster => 21-tdinternal}/03-high-availability.md (100%) rename docs/zh/{10-cluster/04-load-balance.md => 21-tdinternal/05-load-balance.md} (100%) diff --git a/docs/zh/27-train-faq/03-docker.md b/docs/zh/05-get-started/01-docker.md similarity index 100% rename from docs/zh/27-train-faq/03-docker.md rename to docs/zh/05-get-started/01-docker.md diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md new file mode 100644 index 0000000000..e6810ec9b6 --- /dev/null +++ b/docs/zh/05-get-started/03-package.md @@ -0,0 +1,321 @@ +--- +title: 安装包 +description: 使用安装包安装和卸载 +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +:::info +如果您希望对 TDengine 贡献代码或对内部实现感兴趣，请参考我们的 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装. + +::: + +TDengine 开源版本提供 deb 和 rpm 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 deb 支持 Debian/Ubuntu 及衍生系统，rpm 支持 CentOS/RHEL/SUSE 及衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包。 + +## 安装 + + + +可以使用 apt-get 工具从官方仓库安装。 + +**安装包仓库** + +``` +wget -qO - http://repos.taosdata.com/tdengine.key | sudo apt-key add - +echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-stable stable main" | sudo tee /etc/apt/sources.list.d/tdengine-stable.list +``` + +如果安装 Beta 版需要安装包仓库 + +``` +echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-beta beta main" | sudo tee /etc/apt/sources.list.d/tdengine-beta.list +``` + +**使用 apt-get 命令安装** + +``` +sudo apt-get update +apt-cache policy tdengine +sudo apt-get install tdengine +``` + +:::tip +apt-get 方式只适用于 Debian 或 Ubuntu 系统 +:::: + + + +1、从官网下载获得 deb 安装包，例如 TDengine-server-2.4.0.7-Linux-x64.deb； +2、进入到 TDengine-server-2.4.0.7-Linux-x64.deb 安装包所在目录，执行如下的安装命令： + +``` +$ sudo dpkg -i TDengine-server-2.4.0.7-Linux-x64.deb +(Reading database ... 137504 files and directories currently installed.) +Preparing to unpack TDengine-server-2.4.0.7-Linux-x64.deb ... +TDengine is removed successfully! +Unpacking tdengine (2.4.0.7) over (2.4.0.7) ... +Setting up tdengine (2.4.0.7) ... +Start to install TDengine... + +System hostname is: ubuntu-1804 + +Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join +OR leave it blank to build one: + +Enter your email address for priority support or enter empty to skip: +Created symlink /etc/systemd/system/multi-user.target.wants/taosd.service → /etc/systemd/system/taosd.service. + +To configure TDengine : edit /etc/taos/taos.cfg +To start TDengine : sudo systemctl start taosd +To access TDengine : taos -h ubuntu-1804 to login into TDengine server + + +TDengine is installed successfully! +``` + + + + + +1、从官网下载获得 rpm 安装包，例如 TDengine-server-2.4.0.7-Linux-x64.rpm； +2、进入到 TDengine-server-2.4.0.7-Linux-x64.rpm 安装包所在目录，执行如下的安装命令： + +``` +$ sudo rpm -ivh TDengine-server-2.4.0.7-Linux-x64.rpm +Preparing... ################################# [100%] +Updating / installing... + 1:tdengine-2.4.0.7-3 ################################# [100%] +Start to install TDengine... + +System hostname is: centos7 + +Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join +OR leave it blank to build one: + +Enter your email address for priority support or enter empty to skip: + +Created symlink from /etc/systemd/system/multi-user.target.wants/taosd.service to /etc/systemd/system/taosd.service. + +To configure TDengine : edit /etc/taos/taos.cfg +To start TDengine : sudo systemctl start taosd +To access TDengine : taos -h centos7 to login into TDengine server + + +TDengine is installed successfully! +``` + + + + + +1、从官网下载获得 tar.gz 安装包，例如 TDengine-server-2.4.0.7-Linux-x64.tar.gz； +2、进入到 TDengine-server-2.4.0.7-Linux-x64.tar.gz 安装包所在目录，先解压文件后，进入子目录，执行其中的 install.sh 安装脚本： + +``` +$ tar xvzf TDengine-enterprise-server-2.4.0.7-Linux-x64.tar.gz +TDengine-enterprise-server-2.4.0.7/ +TDengine-enterprise-server-2.4.0.7/driver/ +TDengine-enterprise-server-2.4.0.7/driver/vercomp.txt +TDengine-enterprise-server-2.4.0.7/driver/libtaos.so.2.4.0.7 +TDengine-enterprise-server-2.4.0.7/install.sh +TDengine-enterprise-server-2.4.0.7/examples/ +... + +$ ll +total 43816 +drwxrwxr-x 3 ubuntu ubuntu 4096 Feb 22 09:31 ./ +drwxr-xr-x 20 ubuntu ubuntu 4096 Feb 22 09:30 ../ +drwxrwxr-x 4 ubuntu ubuntu 4096 Feb 22 09:30 TDengine-enterprise-server-2.4.0.7/ +-rw-rw-r-- 1 ubuntu ubuntu 44852544 Feb 22 09:31 TDengine-enterprise-server-2.4.0.7-Linux-x64.tar.gz + +$ cd TDengine-enterprise-server-2.4.0.7/ + + $ ll +total 40784 +drwxrwxr-x 4 ubuntu ubuntu 4096 Feb 22 09:30 ./ +drwxrwxr-x 3 ubuntu ubuntu 4096 Feb 22 09:31 ../ +drwxrwxr-x 2 ubuntu ubuntu 4096 Feb 22 09:30 driver/ +drwxrwxr-x 10 ubuntu ubuntu 4096 Feb 22 09:30 examples/ +-rwxrwxr-x 1 ubuntu ubuntu 33294 Feb 22 09:30 install.sh* +-rw-rw-r-- 1 ubuntu ubuntu 41704288 Feb 22 09:30 taos.tar.gz + +$ sudo ./install.sh + +Start to update TDengine... +Created symlink /etc/systemd/system/multi-user.target.wants/taosd.service → /etc/systemd/system/taosd.service. +Nginx for TDengine is updated successfully! + +To configure TDengine : edit /etc/taos/taos.cfg +To configure Taos Adapter (if has) : edit /etc/taos/taosadapter.toml +To start TDengine : sudo systemctl start taosd +To access TDengine : use taos -h ubuntu-1804 in shell OR from http://127.0.0.1:6060 + +TDengine is updated successfully! +Install taoskeeper as a standalone service +taoskeeper is installed, enable it by `systemctl enable taoskeeper` +``` + +:::info +install.sh 安装脚本在执行过程中，会通过命令行交互界面询问一些配置信息。如果希望采取无交互安装方式，那么可以用 -e no 参数来执行 install.sh 脚本。运行 `./install.sh -h` 指令可以查看所有参数的详细说明信息。 + +::: + + + + +:::note +当安装第一个节点时，出现 Enter FQDN：提示的时候，不需要输入任何内容。只有当安装第二个或以后更多的节点时，才需要输入已有集群中任何一个可用节点的 FQDN，支持该新节点加入集群。当然也可以不输入，而是在新节点启动前，配置到新节点的配置文件中。 + +::: + +## 卸载 + + + + +内容TBD + + + + +卸载命令如下: + +``` +$ sudo dpkg -r tdengine +(Reading database ... 137504 files and directories currently installed.) +Removing tdengine (2.4.0.7) ... +TDengine is removed successfully! + +``` + + + + + +卸载命令如下: + +``` +$ sudo rpm -e tdengine +TDengine is removed successfully! +``` + + + + + +卸载命令如下: + +``` +$ rmtaos +Nginx for TDengine is running, stopping it... +TDengine is removed successfully! + +taosKeeper is removed successfully! +``` + + + + +:::info +- TDengine 提供了多种安装包，但最好不要在一个系统上同时使用 tar.gz 安装包和 deb 或 rpm 安装包。否则会相互影响，导致在使用时出现问题。 + +- 对于 deb 包安装后，如果安装目录被手工误删了部分，出现卸载、或重新安装不能成功。此时，需要清除 TDengine 包的安装信息，执行如下命令： + + ``` + $ sudo rm -f /var/lib/dpkg/info/tdengine* + ``` + +然后再重新进行安装就可以了。 + +- 对于 rpm 包安装后，如果安装目录被手工误删了部分，出现卸载、或重新安装不能成功。此时，需要清除 TDengine 包的安装信息，执行如下命令： + + ``` + $ sudo rpm -e --noscripts tdengine + ``` + +然后再重新进行安装就可以了。 + +::: + +## 安装目录说明 + +TDengine 成功安装后，主安装目录是 /usr/local/taos，目录内容如下： + +``` +$ cd /usr/local/taos +$ ll +$ ll +total 28 +drwxr-xr-x 7 root root 4096 Feb 22 09:34 ./ +drwxr-xr-x 12 root root 4096 Feb 22 09:34 ../ +drwxr-xr-x 2 root root 4096 Feb 22 09:34 bin/ +drwxr-xr-x 2 root root 4096 Feb 22 09:34 cfg/ +lrwxrwxrwx 1 root root 13 Feb 22 09:34 data -> /var/lib/taos/ +drwxr-xr-x 2 root root 4096 Feb 22 09:34 driver/ +drwxr-xr-x 10 root root 4096 Feb 22 09:34 examples/ +drwxr-xr-x 2 root root 4096 Feb 22 09:34 include/ +lrwxrwxrwx 1 root root 13 Feb 22 09:34 log -> /var/log/taos/ +``` + +- 自动生成配置文件目录、数据库目录、日志目录。 +- 配置文件缺省目录：/etc/taos/taos.cfg， 软链接到 /usr/local/taos/cfg/taos.cfg； +- 数据库缺省目录：/var/lib/taos， 软链接到 /usr/local/taos/data； +- 日志缺省目录：/var/log/taos， 软链接到 /usr/local/taos/log； +- /usr/local/taos/bin 目录下的可执行文件，会软链接到 /usr/bin 目录下； +- /usr/local/taos/driver 目录下的动态库文件，会软链接到 /usr/lib 目录下； +- /usr/local/taos/include 目录下的头文件，会软链接到到 /usr/include 目录下； + +## 卸载和更新文件说明 + +卸载安装包的时候，将保留配置文件、数据库文件和日志文件，即 /etc/taos/taos.cfg 、 /var/lib/taos 、 /var/log/taos 。如果用户确认后不需保留，可以手工删除，但一定要慎重，因为删除后，数据将永久丢失，不可以恢复！ + +如果是更新安装，当缺省配置文件（ /etc/taos/taos.cfg ）存在时，仍然使用已有的配置文件，安装包中携带的配置文件修改为 taos.cfg.orig 保存在 /usr/local/taos/cfg/ 目录，可以作为设置配置参数的参考样例；如果不存在配置文件，就使用安装包中自带的配置文件。 + +## 启动和停止 + +TDengine 使用 Linux 系统的 systemd/systemctl/service 来管理系统的启动和、停止、重启操作。TDengine 的服务进程是 taosd，默认情况下 TDengine 在系统启动后将自动启动。DBA 可以通过 systemd/systemctl/service 手动操作停止、启动、重新启动服务。 + +以 systemctl 为例，命令如下： + +- 启动服务进程：`systemctl start taosd` + +- 停止服务进程：`systemctl stop taosd` + +- 重启服务进程：`systemctl restart taosd` + +- 查看服务状态：`systemctl status taosd` + +注意：TDengine 在 2.4 版本之后包含一个独立组件 taosAdapter 需要使用 systemctl 命令管理 taosAdapter 服务的启动和停止。 + +如果服务进程处于活动状态，则 status 指令会显示如下的相关信息： + + ``` + Active: active (running) + ``` + +如果后台服务进程处于停止状态，则 status 指令会显示如下的相关信息： + + ``` + Active: inactive (dead) + ``` + +## 升级 +升级分为两个层面：升级安装包 和 升级运行中的实例。 + +升级安装包请遵循前述安装和卸载的步骤先卸载旧版本再安装新版本。 + +升级运行中的实例则要复杂得多，首先请注意版本号，TDengine 的版本号目前分为四段，如 2.4.0.14 和 2.4.0.16，只有前三段版本号一致（即只有第四段版本号不同）才能把一个运行中的实例进行升级。升级步骤如下： +- 停止数据写入 +- 确保所有数据落盘，即写入时序数据库 +- 停止 TDengine 集群 +- 卸载旧版本并安装新版本 +- 重新启动 TDengine 集群 +- 进行简单的查询操作确认旧数据没有丢失 +- 进行简单的写入操作确认 TDengine 集群可用 +- 重新恢复业务数据的写入 + +:::warning +TDengine 不保证低版本能够兼容高版本的数据，所以任何时候都不推荐降级 + +::: \ No newline at end of file diff --git a/docs/zh/05-get-started/06-first-use.md b/docs/zh/05-get-started/06-first-use.md new file mode 100644 index 0000000000..270ea8f7a5 --- /dev/null +++ b/docs/zh/05-get-started/06-first-use.md @@ -0,0 +1,134 @@ +--- +title: 开始使用 +description: '使用 TDengine' +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; +import PkgInstall from "./\_pkg_install.mdx"; +import AptGetInstall from "./\_apt_get_install.mdx"; + +## 启动 + +安装后，请使用 `systemctl` 命令来启动 TDengine 的服务进程。 + +```bash +systemctl start taosd +``` + +检查服务是否正常工作： + +```bash +systemctl status taosd +``` + +如果服务进程处于活动状态，则 status 指令会显示如下的相关信息： + + ``` + Active: active (running) + ``` + +如果后台服务进程处于停止状态，则 status 指令会显示如下的相关信息： + + ``` + Active: inactive (dead) + ``` + +如果 TDengine 服务正常工作，那么您可以通过 TDengine 的命令行程序 `taos` 来访问并体验 TDengine。 + +systemctl 命令汇总： + +- 启动服务进程：`systemctl start taosd` + +- 停止服务进程：`systemctl stop taosd` + +- 重启服务进程：`systemctl restart taosd` + +- 查看服务状态：`systemctl status taosd` + +:::info +- systemctl 命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 sudo 。 +- `systemctl stop taosd` 指令在执行后并不会马上停止 TDengine 服务，而是会等待系统中必要的落盘工作正常完成。在数据量很大的情况下，这可能会消耗较长时间。 +- 如果系统中不支持 `systemd`，也可以用手动运行 `/usr/local/taos/bin/taosd` 方式启动 TDengine 服务。 + +::: + +## TDengine 命令行 (CLI) + +为便于检查 TDengine 的状态，执行数据库 (Database) 的各种即席(Ad Hoc)查询，TDengine 提供一命令行应用程序(以下简称为 TDengine CLI) taos。要进入 TDengine 命令行，您只要在安装有 TDengine 的 Linux 终端执行 `taos` 即可。 + +```bash +taos +``` + +如果连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考 [FAQ](/train-faq/faq) 来解决终端连接服务端失败的问题）。 TDengine CLI 的提示符号如下： + +```cmd +taos> +``` + +在 TDengine CLI 中，用户可以通过 SQL 命令来创建/删除数据库、表等，并进行数据库(database)插入查询操作。在终端中运行的 SQL 语句需要以分号结束来运行。示例： + +```sql +create database demo; +use demo; +create table t (ts timestamp, speed int); +insert into t values ('2019-07-15 00:00:00', 10); +insert into t values ('2019-07-15 01:00:00', 20); +select * from t; + ts | speed | +======================================== + 2019-07-15 00:00:00.000 | 10 | + 2019-07-15 01:00:00.000 | 20 | +Query OK, 2 row(s) in set (0.003128s) +``` + +除执行 SQL 语句外，系统管理员还可以从 TDengine CLI 进行检查系统运行状态、添加删除用户账号等操作。TDengine CLI 连同应用驱动也可以独立安装在 Linux 或 Windows 机器上运行，更多细节请参考 [这里](../reference/taos-shell/) + +## 使用 taosBenchmark 体验写入速度 + +启动 TDengine 的服务，在 Linux 终端执行 `taosBenchmark` （曾命名为 `taosdemo`）： + +```bash +taosBenchmark +``` + +该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupId，groupId 被设置为 1 到 10， location 被设置为 "California.SanFrancisco" 或者 "California.LosAngeles"。 + +这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能，即使在一台普通的 PC 服务器往往也仅需十几秒。 + +taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照 [如何使用 taosBenchmark 对 TDengine 进行性能测试](https://www.taosdata.com/2021/10/09/3111.html)。 + +## 使用 TDengine CLI 体验查询速度 + +使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。 + +查询超级表下记录总条数： + +```sql +taos> select count(*) from test.meters; +``` + +查询 1 亿条记录的平均值、最大值、最小值等： + +```sql +taos> select avg(current), max(voltage), min(phase) from test.meters; +``` + +查询 location="California.SanFrancisco" 的记录总条数： + +```sql +taos> select count(*) from test.meters where location="California.SanFrancisco"; +``` + +查询 groupId=10 的所有记录的平均值、最大值、最小值等： + +```sql +taos> select avg(current), max(voltage), min(phase) from test.meters where groupId=10; +``` + +对表 d10 按 10s 进行平均值、最大值和最小值聚合统计： + +```sql +taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s); +``` diff --git a/docs/zh/10-cluster/02-cluster-mgmt.md b/docs/zh/10-cluster/02-cluster-mgmt.md deleted file mode 100644 index 5c490516f0..0000000000 --- a/docs/zh/10-cluster/02-cluster-mgmt.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -title: 数据节点管理 ---- - -上面已经介绍如何从零开始搭建集群。集群组建完成后，可以随时查看集群中当前的数据节点的状态，还可以添加新的数据节点进行扩容，删除数据节点，甚至手动进行数据节点之间的负载均衡操作。 - -:::note - -以下所有执行命令的操作需要先登陆进 TDengine 系统，必要时请使用 root 权限。 - -::: - -## 查看数据节点 - -启动 TDengine CLI 程序 taos，然后执行： - -```sql -SHOW DNODES; -``` - -它将列出集群中所有的 dnode，每个 dnode 的 ID，end_point（fqdn:port），状态（ready，offline 等），vnode 数目，还未使用的 vnode 数目等信息。在添加或删除一个数据节点后，可以使用该命令查看。 - -输出如下（具体内容仅供参考，取决于实际的集群配置） - -``` -taos> show dnodes; - id | endpoint | vnodes | support_vnodes | status | create_time | note | -============================================================================================================================================ - 1 | trd01:6030 | 100 | 1024 | ready | 2022-07-15 16:47:47.726 | | -Query OK, 1 rows affected (0.006684s) -``` - -## 查看虚拟节点组 - -为充分利用多核技术，并提供横向扩展能力，数据需要分片处理。因此 TDengine 会将一个 DB 的数据切分成多份，存放在多个 vnode 里。这些 vnode 可能分布在多个数据节点 dnode 里，这样就实现了水平扩展。一个 vnode 仅仅属于一个 DB，但一个 DB 可以有多个 vnode。vnode 所在的数据节点是 mnode 根据当前系统资源的情况，自动进行分配的，无需任何人工干预。 - -启动 CLI 程序 taos，然后执行： - -```sql -USE SOME_DATABASE; -SHOW VGROUPS; -``` - -输出如下（具体内容仅供参考，取决于实际的集群配置） - -``` -taos> use db; -Database changed. - -taos> show vgroups; - vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | status | nfiles | file_size | tsma | -================================================================================================================================================================================================ - 2 | db | 0 | 1 | leader | NULL | NULL | NULL | NULL | NULL | NULL | NULL | 0 | - 3 | db | 0 | 1 | leader | NULL | NULL | NULL | NULL | NULL | NULL | NULL | 0 | - 4 | db | 0 | 1 | leader | NULL | NULL | NULL | NULL | NULL | NULL | NULL | 0 | -Query OK, 8 row(s) in set (0.001154s) -``` - -## 添加数据节点 - -启动 CLI 程序 taos，然后执行： - -```sql -CREATE DNODE "fqdn:port"; -``` - -将新数据节点的 End Point 添加进集群的 EP 列表。“fqdn:port“需要用双引号引起来，否则出错。一个数据节点对外服务的 fqdn 和 port 可以通过配置文件 taos.cfg 进行配置，缺省是自动获取。【强烈不建议用自动获取方式来配置 FQDN，可能导致生成的数据节点的 End Point 不是所期望的】 - -然后启动新加入的数据节点的 taosd 进程，再通过 taos 查看数据节点状态： - -``` -taos> show dnodes; - id | endpoint | vnodes | support_vnodes | status | create_time | note | -============================================================================================================================================ - 1 | localhost:6030 | 100 | 1024 | ready | 2022-07-15 16:47:47.726 | | - 2 | localhost:7030 | 0 | 1024 | ready | 2022-07-15 16:56:13.670 | | -Query OK, 2 rows affected (0.007031s) -``` - -从中可以看到两个 dnode 状态都为 ready - -## 删除数据节点 - -先停止要删除的数据节点的 taosd 进程，然后启动 CLI 程序 taos，执行： - -```sql -DROP DNODE "fqdn:port"; -``` - -或者 - -```sql -DROP DNODE dnodeId; -``` - -通过 “fqdn:port” 或 dnodeID 来指定一个具体的节点都是可以的。其中 fqdn 是被删除的节点的 FQDN，port 是其对外服务器的端口号；dnodeID 可以通过 SHOW DNODES 获得。 - -:::warning - -数据节点一旦被 drop 之后，不能重新加入集群。需要将此节点重新部署（清空数据文件夹）。集群在完成 `drop dnode` 操作之前，会将该 dnode 的数据迁移走。 -请注意 `drop dnode` 和 停止 taosd 进程是两个不同的概念，不要混淆：因为删除 dnode 之前要执行迁移数据的操作，因此被删除的 dnode 必须保持在线状态。待删除操作结束之后，才能停止 taosd 进程。 -一个数据节点被 drop 之后，其他节点都会感知到这个 dnodeID 的删除操作，任何集群中的节点都不会再接收此 dnodeID 的请求。 -dnodeID 是集群自动分配的，不得人工指定。它在生成时是递增的，不会重复。 - -::: diff --git a/docs/zh/10-cluster/_category_.yml b/docs/zh/10-cluster/_category_.yml deleted file mode 100644 index 3cee5ce4cd..0000000000 --- a/docs/zh/10-cluster/_category_.yml +++ /dev/null @@ -1 +0,0 @@ -label: 集群管理 diff --git a/docs/zh/10-cluster/01-deploy.md b/docs/zh/10-deployment/01-deploy.md similarity index 64% rename from docs/zh/10-cluster/01-deploy.md rename to docs/zh/10-deployment/01-deploy.md index cd19f90ba1..947ca61fe1 100644 --- a/docs/zh/10-cluster/01-deploy.md +++ b/docs/zh/10-deployment/01-deploy.md @@ -1,5 +1,5 @@ --- -title: 集群部署 +title: 集群部署和管理 --- ## 准备工作 @@ -91,7 +91,7 @@ taos> 上述命令里，可以看到刚启动的数据节点的 End Point 是：h1.taos.com:6030，就是这个新集群的 firstEp。 -### 启动后续数据节点 +### 添加数据节点 将后续的数据节点添加到现有集群，具体有以下几步： @@ -125,3 +125,75 @@ firstEp 这个参数仅仅在该数据节点首次加入集群时有作用，加 两个没有配置 firstEp 参数的数据节点 dnode 启动后，会独立运行起来。这个时候，无法将其中一个数据节点加入到另外一个数据节点，形成集群。无法将两个独立的集群合并成为新的集群。 ::: + +## 查看数据节点 + +启动 TDengine CLI 程序 taos，然后执行： + +```sql +SHOW DNODES; +``` + +它将列出集群中所有的 dnode，每个 dnode 的 ID，end_point（fqdn:port），状态（ready，offline 等），vnode 数目，还未使用的 vnode 数目等信息。在添加或删除一个数据节点后，可以使用该命令查看。 + +输出如下（具体内容仅供参考，取决于实际的集群配置） + +``` +taos> show dnodes; + id | endpoint | vnodes | support_vnodes | status | create_time | note | +============================================================================================================================================ + 1 | trd01:6030 | 100 | 1024 | ready | 2022-07-15 16:47:47.726 | | +Query OK, 1 rows affected (0.006684s) +``` + +## 查看虚拟节点组 + +为充分利用多核技术，并提供横向扩展能力，数据需要分片处理。因此 TDengine 会将一个 DB 的数据切分成多份，存放在多个 vnode 里。这些 vnode 可能分布在多个数据节点 dnode 里，这样就实现了水平扩展。一个 vnode 仅仅属于一个 DB，但一个 DB 可以有多个 vnode。vnode 所在的数据节点是 mnode 根据当前系统资源的情况，自动进行分配的，无需任何人工干预。 + +启动 CLI 程序 taos，然后执行： + +```sql +USE SOME_DATABASE; +SHOW VGROUPS; +``` + +输出如下（具体内容仅供参考，取决于实际的集群配置） + +``` +taos> use db; +Database changed. + +taos> show vgroups; + vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | status | nfiles | file_size | tsma | +================================================================================================================================================================================================ + 2 | db | 0 | 1 | leader | NULL | NULL | NULL | NULL | NULL | NULL | NULL | 0 | + 3 | db | 0 | 1 | leader | NULL | NULL | NULL | NULL | NULL | NULL | NULL | 0 | + 4 | db | 0 | 1 | leader | NULL | NULL | NULL | NULL | NULL | NULL | NULL | 0 | +Query OK, 8 row(s) in set (0.001154s) +``` + +## 删除数据节点 + +先停止要删除的数据节点的 taosd 进程，然后启动 CLI 程序 taos，执行： + +```sql +DROP DNODE "fqdn:port"; +``` + +或者 + +```sql +DROP DNODE dnodeId; +``` + +通过 “fqdn:port” 或 dnodeID 来指定一个具体的节点都是可以的。其中 fqdn 是被删除的节点的 FQDN，port 是其对外服务器的端口号；dnodeID 可以通过 SHOW DNODES 获得。 + +:::warning + +数据节点一旦被 drop 之后，不能重新加入集群。需要将此节点重新部署（清空数据文件夹）。集群在完成 `drop dnode` 操作之前，会将该 dnode 的数据迁移走。 +请注意 `drop dnode` 和 停止 taosd 进程是两个不同的概念，不要混淆：因为删除 dnode 之前要执行迁移数据的操作，因此被删除的 dnode 必须保持在线状态。待删除操作结束之后，才能停止 taosd 进程。 +一个数据节点被 drop 之后，其他节点都会感知到这个 dnodeID 的删除操作，任何集群中的节点都不会再接收此 dnodeID 的请求。 +dnodeID 是集群自动分配的，不得人工指定。它在生成时是递增的，不会重复。 + +::: + diff --git a/docs/zh/10-deployment/03-k8s.md b/docs/zh/10-deployment/03-k8s.md new file mode 100644 index 0000000000..2ca28abe86 --- /dev/null +++ b/docs/zh/10-deployment/03-k8s.md @@ -0,0 +1,451 @@ +--- +title: 在 Kubernetes 上部署 TDengine 集群 +--- + +## 配置 ConfigMap + +为 TDengine 创建 `taoscfg.yaml`，此文件中的配置将作为环境变量传入 TDengine 镜像，更新此配置将导致所有 TDengine POD 重启。 + +```yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: taoscfg + labels: + app: tdengine +data: + CLUSTER: "1" + TAOS_KEEP: "3650" + TAOS_DEBUG_FLAG: "135" +``` + +## 配置服务 + +创建一个 service 配置文件：`taosd-service.yaml`，服务名称 `metadata.name` (此处为 "taosd") 将在下一步中使用到。添加 TDengine 所用到的所有端口： + +```yaml +--- +apiVersion: v1 +kind: Service +metadata: + name: "taosd" + labels: + app: "tdengine" +spec: + ports: + - name: tcp6030 + protocol: "TCP" + port: 6030 + - name: tcp6035 + protocol: "TCP" + port: 6035 + - name: tcp6041 + protocol: "TCP" + port: 6041 + - name: udp6030 + protocol: "UDP" + port: 6030 + - name: udp6031 + protocol: "UDP" + port: 6031 + - name: udp6032 + protocol: "UDP" + port: 6032 + - name: udp6033 + protocol: "UDP" + port: 6033 + - name: udp6034 + protocol: "UDP" + port: 6034 + - name: udp6035 + protocol: "UDP" + port: 6035 + - name: udp6036 + protocol: "UDP" + port: 6036 + - name: udp6037 + protocol: "UDP" + port: 6037 + - name: udp6038 + protocol: "UDP" + port: 6038 + - name: udp6039 + protocol: "UDP" + port: 6039 + - name: udp6040 + protocol: "UDP" + port: 6040 + selector: + app: "tdengine" +``` + +## 有状态服务 StatefulSet + +根据 Kubernetes 对各类部署的说明，我们将使用 StatefulSet 作为 TDengine 的服务类型，创建文件 `tdengine.yaml`： + +```yaml +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: "tdengine" + labels: + app: "tdengine" +spec: + serviceName: "taosd" + replicas: 2 + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app: "tdengine" + template: + metadata: + name: "tdengine" + labels: + app: "tdengine" + spec: + containers: + - name: "tdengine" + image: "zitsen/taosd:develop" + imagePullPolicy: "Always" + envFrom: + - configMapRef: + name: taoscfg + ports: + - name: tcp6030 + protocol: "TCP" + containerPort: 6030 + - name: tcp6035 + protocol: "TCP" + containerPort: 6035 + - name: tcp6041 + protocol: "TCP" + containerPort: 6041 + - name: udp6030 + protocol: "UDP" + containerPort: 6030 + - name: udp6031 + protocol: "UDP" + containerPort: 6031 + - name: udp6032 + protocol: "UDP" + containerPort: 6032 + - name: udp6033 + protocol: "UDP" + containerPort: 6033 + - name: udp6034 + protocol: "UDP" + containerPort: 6034 + - name: udp6035 + protocol: "UDP" + containerPort: 6035 + - name: udp6036 + protocol: "UDP" + containerPort: 6036 + - name: udp6037 + protocol: "UDP" + containerPort: 6037 + - name: udp6038 + protocol: "UDP" + containerPort: 6038 + - name: udp6039 + protocol: "UDP" + containerPort: 6039 + - name: udp6040 + protocol: "UDP" + containerPort: 6040 + env: + # POD_NAME for FQDN config + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # SERVICE_NAME and NAMESPACE for fqdn resolve + - name: SERVICE_NAME + value: "taosd" + - name: STS_NAME + value: "tdengine" + - name: STS_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # TZ for timezone settings, we recommend to always set it. + - name: TZ + value: "Asia/Shanghai" + # TAOS_ prefix will configured in taos.cfg, strip prefix and camelCase. + - name: TAOS_SERVER_PORT + value: "6030" + # Must set if you want a cluster. + - name: TAOS_FIRST_EP + value: "$(STS_NAME)-0.$(SERVICE_NAME).$(STS_NAMESPACE).svc.cluster.local:$(TAOS_SERVER_PORT)" + # TAOS_FQND should always be setted in k8s env. + - name: TAOS_FQDN + value: "$(POD_NAME).$(SERVICE_NAME).$(STS_NAMESPACE).svc.cluster.local" + volumeMounts: + - name: taosdata + mountPath: /var/lib/taos + readinessProbe: + exec: + command: + - taos + - -s + - "show mnodes" + initialDelaySeconds: 5 + timeoutSeconds: 5000 + livenessProbe: + tcpSocket: + port: 6030 + initialDelaySeconds: 15 + periodSeconds: 20 + volumeClaimTemplates: + - metadata: + name: taosdata + spec: + accessModes: + - "ReadWriteOnce" + storageClassName: "csi-rbd-sc" + resources: + requests: + storage: "10Gi" +``` + +## 启动集群 + +将前述三个文件添加到 Kubernetes 集群中： + +```bash +kubectl apply -f taoscfg.yaml +kubectl apply -f taosd-service.yaml +kubectl apply -f tdengine.yaml + +``` + +上面的配置将生成一个两节点的 TDengine 集群，dnode 是自动配置的，可以使用 `show dnodes` 命令查看当前集群的节点： + +```bash +kubectl exec -i -t tdengine-0 -- taos -s "show dnodes" +kubectl exec -i -t tdengine-1 -- taos -s "show dnodes" + +``` + +输出如下： + +``` +Welcome to the TDengine shell from Linux, Client Version:2.1.1.0 +Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. + +taos> show dnodes + id | end_point | vnodes | cores | status | role | create_time | offline reason | +====================================================================================================================================== + 1 | tdengine-0.taosd.default.sv... | 1 | 40 | ready | any | 2021-06-01 17:13:24.181 | | + 2 | tdengine-1.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 17:14:09.257 | | +Query OK, 2 row(s) in set (0.000997s) + +``` + +## 集群扩容 + +TDengine 集群支持自动扩容： + +```bash +kubectl scale statefulsets tdengine --replicas=4 + +``` + +上面命令行中参数 `--replica=4` 表示要将 TDengine 集群扩容到 4 个节点，执行后首先检查 POD 的状态： + +```bash +kubectl get pods -l app=tdengine + +``` + +输出如下： + +``` +NAME READY STATUS RESTARTS AGE +tdengine-0 1/1 Running 0 161m +tdengine-1 1/1 Running 0 161m +tdengine-2 1/1 Running 0 32m +tdengine-3 1/1 Running 0 32m + +``` + +此时 POD 的状态仍然是 Running，TDengine 集群中的 dnode 状态要等 POD 状态为 `ready` 之后才能看到： + +```bash +kubectl exec -i -t tdengine-0 -- taos -s "show dnodes" + +``` + +扩容后的四节点 TDengine 集群的 dnode 列表: + +``` +Welcome to the TDengine shell from Linux, Client Version:2.1.1.0 +Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. + +taos> show dnodes + id | end_point | vnodes | cores | status | role | create_time | offline reason | +====================================================================================================================================== + 1 | tdengine-0.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 11:58:12.915 | | + 2 | tdengine-1.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 11:58:33.127 | | + 3 | tdengine-2.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 14:07:27.078 | | + 4 | tdengine-3.taosd.default.sv... | 1 | 40 | ready | any | 2021-06-01 14:07:48.362 | | +Query OK, 4 row(s) in set (0.001293s) + +``` + +## 集群缩容 + +TDengine 的缩容并没有自动化，我们尝试将一个三节点集群缩容到两节点。 + +首先，确认一个三节点 TDengine 集群正常工作，在 TDengine CLI 中查看 dnode 的状态： + +```bash +taos> show dnodes + id | end_point | vnodes | cores | status | role | create_time | offline reason | +====================================================================================================================================== + 1 | tdengine-0.taosd.default.sv... | 1 | 40 | ready | any | 2021-06-01 16:27:24.852 | | + 2 | tdengine-1.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 16:27:53.339 | | + 3 | tdengine-2.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 16:28:49.787 | | +Query OK, 3 row(s) in set (0.001101s) + +``` + +想要安全的缩容，首先需要将节点从 dnode 列表中移除，也即从集群中移除： + +```bash +kubectl exec -i -t tdengine-0 -- taos -s "drop dnode 'tdengine-2.taosd.default.svc.cluster.local:6030'" + +``` + +通过 `show dondes` 命令确认移除成功后，移除相应的 POD： + +```bash +kubectl scale statefulsets tdengine --replicas=2 + +``` + +最后一个 POD 会被删除，使用 `kubectl get pods -l app=tdengine` 查看集群状态： + +``` +NAME READY STATUS RESTARTS AGE +tdengine-0 1/1 Running 0 3h40m +tdengine-1 1/1 Running 0 3h40m + +``` + +POD 删除后，需要手动删除 PVC，否则下次扩容时会继续使用以前的数据导致无法正常加入集群。 + +```bash +kubectl delete pvc taosdata-tdengine-2 + +``` + +此时的集群状态是安全的，需要时还可以再次进行扩容： + +```bash +kubectl scale statefulsets tdengine --replicas=3 + + +``` + +`show dnodes` 输出如下： + +``` +taos> show dnodes + id | end_point | vnodes | cores | status | role | create_time | offline reason | +====================================================================================================================================== + 1 | tdengine-0.taosd.default.sv... | 1 | 40 | ready | any | 2021-06-01 16:27:24.852 | | + 2 | tdengine-1.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 16:27:53.339 | | + 4 | tdengine-2.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 16:40:49.177 | | + + +``` + +## 删除集群 + +完整移除 TDengine 集群，需要分别清理 statefulset、svc、configmap、pvc。 + +```bash +kubectl delete statefulset -l app=tdengine +kubectl delete svc -l app=tdengine +kubectl delete pvc -l app=tdengine +kubectl delete configmap taoscfg + +``` + +## 常见错误 + +### 错误一 + +扩容到四节点之后缩容到两节点，删除的 POD 会进入 offline 状态： + +``` +Welcome to the TDengine shell from Linux, Client Version:2.1.1.0 +Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. + +taos> show dnodes + id | end_point | vnodes | cores | status | role | create_time | offline reason | +====================================================================================================================================== + 1 | tdengine-0.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 11:58:12.915 | | + 2 | tdengine-1.taosd.default.sv... | 0 | 40 | ready | any | 2021-06-01 11:58:33.127 | | + 3 | tdengine-2.taosd.default.sv... | 0 | 40 | offline | any | 2021-06-01 14:07:27.078 | status msg timeout | + 4 | tdengine-3.taosd.default.sv... | 1 | 40 | offline | any | 2021-06-01 14:07:48.362 | status msg timeout | +Query OK, 4 row(s) in set (0.001236s) + + +``` + +但 `drop dnode` 的行为按不会按照预期进行，且下次集群重启后，所有的 dnode 节点将无法启动 dropping 状态无法退出。 + +### 错误二 + +TDengine集群会持有 replica 参数，如果缩容后的节点数小于这个值，集群将无法使用： + +创建一个库使用 replica 参数为 2，插入部分数据： + +```bash +kubectl exec -i -t tdengine-0 -- \ + taos -s \ + "create database if not exists test replica 2; + use test; + create table if not exists t1(ts timestamp, n int); + insert into t1 values(now, 1)(now+1s, 2);" + + +``` + +缩容到单节点： + +```bash +kubectl scale statefulsets tdengine --replicas=1 + +``` + +在taos shell中的所有数据库操作将无法成功。 + +``` +taos> show dnodes; + id | end_point | vnodes | cores | status | role | create_time | offline reason | +====================================================================================================================================== + 1 | tdengine-0.taosd.default.sv... | 2 | 40 | ready | any | 2021-06-01 15:55:52.562 | | + 2 | tdengine-1.taosd.default.sv... | 1 | 40 | offline | any | 2021-06-01 15:56:07.212 | status msg timeout | +Query OK, 2 row(s) in set (0.000845s) + +taos> show dnodes; + id | end_point | vnodes | cores | status | role | create_time | offline reason | +====================================================================================================================================== + 1 | tdengine-0.taosd.default.sv... | 2 | 40 | ready | any | 2021-06-01 15:55:52.562 | | + 2 | tdengine-1.taosd.default.sv... | 1 | 40 | offline | any | 2021-06-01 15:56:07.212 | status msg timeout | +Query OK, 2 row(s) in set (0.000837s) + +taos> use test; +Database changed. + +taos> insert into t1 values(now, 3); + +DB error: Unable to resolve FQDN (0.013874s) + +``` \ No newline at end of file diff --git a/docs/zh/10-deployment/05-helm.md b/docs/zh/10-deployment/05-helm.md new file mode 100644 index 0000000000..926b71c500 --- /dev/null +++ b/docs/zh/10-deployment/05-helm.md @@ -0,0 +1,415 @@ +--- +title: 使用 Helm 部署 TDengine 集群 +--- + +Helm 是 Kubernetes 的包管理器，上一节使用 Kubernets 部署 TDengine 集群的操作已经足够简单，但 Helm 依然可以提供更强大的能力。 + +## 安装 Helm + +```bash +curl -fsSL -o get_helm.sh \ + https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 +chmod +x get_helm.sh +./get_helm.sh + +``` + +Helm 会使用 kubectl 和 kubeconfig 的配置来操作 Kubernetes，可以参考 Rancher 安装 Kubernetes 的配置来进行设置。 + +## 安装 TDengine Chart + +TDengine Chart 尚未发布到 Helm 仓库，当前可以从GitHub直接下载： + +```bash +wget https://github.com/taosdata/TDengine-Operator/raw/main/helm/tdengine-0.3.0.tgz + +``` + +获取当前Kubernetes的存储类： + +```bash +kubectl get storageclass + +``` + +在 minikube 默认为 standard. + +之后，使用helm命令安装： + +```bash +helm install tdengine tdengine-0.3.0.tgz \ + --set storage.className= + +``` + +在 minikube 环境下，可以设置一个较小的容量避免超出磁盘可用空间： + +```bash +helm install tdengine tdengine-0.3.0.tgz \ + --set storage.className=standard \ + --set storage.dataSize=2Gi \ + --set storage.logSize=10Mi + +``` + +部署成功后，TDengine Chart将会输出操作TDengine的说明： + +```bash +export POD_NAME=$(kubectl get pods --namespace default \ + -l "app.kubernetes.io/name=tdengine,app.kubernetes.io/instance=tdengine" \ + -o jsonpath="{.items[0].metadata.name}") +kubectl --namespace default exec $POD_NAME -- taos -s "show dnodes; show mnodes" +kubectl --namespace default exec -it $POD_NAME -- taos + +``` + +可以创建一个表进行测试： + +```bash +kubectl --namespace default exec $POD_NAME -- \ + taos -s "create database test; + use test; + create table t1 (ts timestamp, n int); + insert into t1 values(now, 1)(now + 1s, 2); + select * from t1;" + +``` + +## 配置 Values + +TDengine 支持 `values.yaml` 自定义。 + +通过 helm show values可以获取TDengine Chart支持的全部values列表： +```bash +helm show values tdengine-0.3.0.tgz + +``` + +你可以将结果保存为 values.yaml，之后可以修改其中的各项参数，如 replica 数量，存储类名称，容量大小，TDengine 配置等，然后使用如下命令安装TDengine集群： + +```bash +helm install tdengine tdengine-0.3.0.tgz -f values.yaml + +``` + +全部参数如下： + +```yaml +# Default values for tdengine. +# This is a YAML-formatted file. +# Declare variables to be passed into helm templates. + +replicaCount: 1 + +image: + prefix: tdengine/tdengine + #pullPolicy: Always + # Overrides the image tag whose default is the chart appVersion. + #tag: "2.4.0.5" + +service: + # ClusterIP is the default service type, use NodeIP only if you know what you are doing. + type: ClusterIP + ports: + # TCP range required + tcp: [6030,6031,6032,6033,6034, 6035,6036,6037,6038, 6039, 6040, 6041, 6042, 6043, 6044, 6045, 6060] + # UDP range 6030-6039 + udp: [6030, 6031, 6032, 6033, 6034, 6035, 6036, 6037, 6038, 6039] + +arbitrator: true + +# Set timezone here, not in taoscfg +timezone: "Asia/Shanghai" + +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +storage: + # Set storageClassName for pvc. K8s use default storage class if not set. + # + className: "" + dataSize: "100Gi" + logSize: "10Gi" + +nodeSelectors: + taosd: + # node selectors + +clusterDomainSuffix: "" +# Config settings in taos.cfg file. +# +# The helm/k8s support will use environment variables for taos.cfg, +# converting an upper-snake-cased variable like `TAOS_DEBUG_FLAG`, +# to a camelCase taos config variable `debugFlag`. +# +# See the variable list at https://www.taosdata.com/cn/documentation/administrator . +# +# Note: +# 1. firstEp/secondEp: should not be setted here, it's auto generated at scale-up. +# 2. serverPort: should not be setted, we'll use the default 6030 in many places. +# 3. fqdn: will be auto generated in kubenetes, user should not care about it. +# 4. role: currently role is not supported - every node is able to be mnode and vnode. +# +# Btw, keep quotes "" around the value like below, even the value will be number or not. +taoscfg: + + # number of replications, for cluster only + TAOS_REPLICA: "1" + + # number of management nodes in the system + TAOS_NUM_OF_MNODES: "1" + + # number of days per DB file + # TAOS_DAYS: "10" + + # number of days to keep DB file, default is 10 years. + #TAOS_KEEP: "3650" + + # cache block size (Mbyte) + #TAOS_CACHE: "16" + + # number of cache blocks per vnode + #TAOS_BLOCKS: "6" + + # minimum rows of records in file block + #TAOS_MIN_ROWS: "100" + + # maximum rows of records in file block + #TAOS_MAX_ROWS: "4096" + + # + # TAOS_NUM_OF_THREADS_PER_CORE: number of threads per CPU core + #TAOS_NUM_OF_THREADS_PER_CORE: "1.0" + + # + # TAOS_NUM_OF_COMMIT_THREADS: number of threads to commit cache data + #TAOS_NUM_OF_COMMIT_THREADS: "4" + + # + # TAOS_RATIO_OF_QUERY_CORES: + # the proportion of total CPU cores available for query processing + # 2.0: the query threads will be set to double of the CPU cores. + # 1.0: all CPU cores are available for query processing [default]. + # 0.5: only half of the CPU cores are available for query. + # 0.0: only one core available. + #TAOS_RATIO_OF_QUERY_CORES: "1.0" + + # + # TAOS_KEEP_COLUMN_NAME: + # the last_row/first/last aggregator will not change the original column name in the result fields + #TAOS_KEEP_COLUMN_NAME: "0" + + # enable/disable backuping vnode directory when removing vnode + #TAOS_VNODE_BAK: "1" + + # enable/disable installation / usage report + #TAOS_TELEMETRY_REPORTING: "1" + + # enable/disable load balancing + #TAOS_BALANCE: "1" + + # max timer control blocks + #TAOS_MAX_TMR_CTRL: "512" + + # time interval of system monitor, seconds + #TAOS_MONITOR_INTERVAL: "30" + + # number of seconds allowed for a dnode to be offline, for cluster only + #TAOS_OFFLINE_THRESHOLD: "8640000" + + # RPC re-try timer, millisecond + #TAOS_RPC_TIMER: "1000" + + # RPC maximum time for ack, seconds. + #TAOS_RPC_MAX_TIME: "600" + + # time interval of dnode status reporting to mnode, seconds, for cluster only + #TAOS_STATUS_INTERVAL: "1" + + # time interval of heart beat from shell to dnode, seconds + #TAOS_SHELL_ACTIVITY_TIMER: "3" + + # minimum sliding window time, milli-second + #TAOS_MIN_SLIDING_TIME: "10" + + # minimum time window, milli-second + #TAOS_MIN_INTERVAL_TIME: "10" + + # maximum delay before launching a stream computation, milli-second + #TAOS_MAX_STREAM_COMP_DELAY: "20000" + + # maximum delay before launching a stream computation for the first time, milli-second + #TAOS_MAX_FIRST_STREAM_COMP_DELAY: "10000" + + # retry delay when a stream computation fails, milli-second + #TAOS_RETRY_STREAM_COMP_DELAY: "10" + + # the delayed time for launching a stream computation, from 0.1(default, 10% of whole computing time window) to 0.9 + #TAOS_STREAM_COMP_DELAY_RATIO: "0.1" + + # max number of vgroups per db, 0 means configured automatically + #TAOS_MAX_VGROUPS_PER_DB: "0" + + # max number of tables per vnode + #TAOS_MAX_TABLES_PER_VNODE: "1000000" + + # the number of acknowledgments required for successful data writing + #TAOS_QUORUM: "1" + + # enable/disable compression + #TAOS_COMP: "2" + + # write ahead log (WAL) level, 0: no wal; 1: write wal, but no fysnc; 2: write wal, and call fsync + #TAOS_WAL_LEVEL: "1" + + # if walLevel is set to 2, the cycle of fsync being executed, if set to 0, fsync is called right away + #TAOS_FSYNC: "3000" + + # the compressed rpc message, option: + # -1 (no compression) + # 0 (all message compressed), + # > 0 (rpc message body which larger than this value will be compressed) + #TAOS_COMPRESS_MSG_SIZE: "-1" + + # max length of an SQL + #TAOS_MAX_SQL_LENGTH: "1048576" + + # the maximum number of records allowed for super table time sorting + #TAOS_MAX_NUM_OF_ORDERED_RES: "100000" + + # max number of connections allowed in dnode + #TAOS_MAX_SHELL_CONNS: "5000" + + # max number of connections allowed in client + #TAOS_MAX_CONNECTIONS: "5000" + + # stop writing logs when the disk size of the log folder is less than this value + #TAOS_MINIMAL_LOG_DIR_G_B: "0.1" + + # stop writing temporary files when the disk size of the tmp folder is less than this value + #TAOS_MINIMAL_TMP_DIR_G_B: "0.1" + + # if disk free space is less than this value, taosd service exit directly within startup process + #TAOS_MINIMAL_DATA_DIR_G_B: "0.1" + + # One mnode is equal to the number of vnode consumed + #TAOS_MNODE_EQUAL_VNODE_NUM: "4" + + # enbale/disable http service + #TAOS_HTTP: "1" + + # enable/disable system monitor + #TAOS_MONITOR: "1" + + # enable/disable recording the SQL statements via restful interface + #TAOS_HTTP_ENABLE_RECORD_SQL: "0" + + # number of threads used to process http requests + #TAOS_HTTP_MAX_THREADS: "2" + + # maximum number of rows returned by the restful interface + #TAOS_RESTFUL_ROW_LIMIT: "10240" + + # The following parameter is used to limit the maximum number of lines in log files. + # max number of lines per log filters + # numOfLogLines 10000000 + + # enable/disable async log + #TAOS_ASYNC_LOG: "0" + + # + # time of keeping log files, days + #TAOS_LOG_KEEP_DAYS: "0" + + # The following parameters are used for debug purpose only. + # debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR + # 131: output warning and error + # 135: output debug, warning and error + # 143: output trace, debug, warning and error to log + # 199: output debug, warning and error to both screen and file + # 207: output trace, debug, warning and error to both screen and file + # + # debug flag for all log type, take effect when non-zero value\ + #TAOS_DEBUG_FLAG: "143" + + # enable/disable recording the SQL in taos client + #TAOS_ENABLE_RECORD_SQL: "0" + + # generate core file when service crash + #TAOS_ENABLE_CORE_FILE: "1" + + # maximum display width of binary and nchar fields in the shell. The parts exceeding this limit will be hidden + #TAOS_MAX_BINARY_DISPLAY_WIDTH: "30" + + # enable/disable stream (continuous query) + #TAOS_STREAM: "1" + + # in retrieve blocking model, only in 50% query threads will be used in query processing in dnode + #TAOS_RETRIEVE_BLOCKING_MODEL: "0" + + # the maximum allowed query buffer size in MB during query processing for each data node + # -1 no limit (default) + # 0 no query allowed, queries are disabled + #TAOS_QUERY_BUFFER_SIZE: "-1" + +``` + +## 扩容 + +关于扩容可参考上一节的说明，有一些额外的操作需要从helm的部署中获取。 + +首先，从部署中获取StatefulSet的名称。 + +```bash +export STS_NAME=$(kubectl get statefulset \ + -l "app.kubernetes.io/name=tdengine" \ + -o jsonpath="{.items[0].metadata.name}") + +``` + +扩容操作极其简单，增加replica即可。以下命令将TDengine扩充到三节点： + +```bash +kubectl scale --replicas 3 statefulset/$STS_NAME + +``` + +使用命令 `show dnodes` 和 `show mnodes` 检查是否扩容成功。 + +## 缩容 + +:::warning +缩容操作并没有完整测试，可能造成数据风险，请谨慎使用。 + +::: + +获取需要缩容的dnode列表，并手动Drop。 + +```bash +kubectl --namespace default exec $POD_NAME -- \ + cat /var/lib/taos/dnode/dnodeEps.json \ + | jq '.dnodeInfos[1:] |map(.dnodeFqdn + ":" + (.dnodePort|tostring)) | .[]' -r +kubectl --namespace default exec $POD_NAME -- taos -s "show dnodes" +kubectl --namespace default exec $POD_NAME -- taos -s 'drop dnode ""' + +``` + +## 删除集群 + +Helm管理下，清理操作也变得简单： + +```bash +helm uninstall tdengine + +``` + +但Helm也不会自动移除PVC，需要手动获取PVC然后删除掉。 \ No newline at end of file diff --git a/docs/zh/10-deployment/_category_.yml b/docs/zh/10-deployment/_category_.yml new file mode 100644 index 0000000000..38363bd571 --- /dev/null +++ b/docs/zh/10-deployment/_category_.yml @@ -0,0 +1 @@ +label: 部署集群 diff --git a/docs/zh/10-cluster/index.md b/docs/zh/10-deployment/index.md similarity index 82% rename from docs/zh/10-cluster/index.md rename to docs/zh/10-deployment/index.md index ef2a7253c9..96ac7b176d 100644 --- a/docs/zh/10-cluster/index.md +++ b/docs/zh/10-deployment/index.md @@ -1,10 +1,10 @@ --- -title: 集群管理 +title: 部署集群 --- TDengine 支持集群，提供水平扩展的能力。如果需要获得更高的处理能力，只需要多增加节点即可。TDengine 采用虚拟节点技术，将一个节点虚拟化为多个虚拟节点，以实现负载均衡。同时，TDengine可以将多个节点上的虚拟节点组成虚拟节点组，通过多副本机制，以保证供系统的高可用。TDengine的集群功能完全开源。 -本章节主要介绍集群的部署、维护，以及如何实现高可用和负载均衡。 +本章节主要介绍如何在主机上人工部署集群，以及如何使用 Kubernetes 和 Helm部署集群。 ```mdx-code-block import DocCardList from '@theme/DocCardList'; diff --git a/docs/zh/10-cluster/03-high-availability.md b/docs/zh/21-tdinternal/03-high-availability.md similarity index 100% rename from docs/zh/10-cluster/03-high-availability.md rename to docs/zh/21-tdinternal/03-high-availability.md diff --git a/docs/zh/10-cluster/04-load-balance.md b/docs/zh/21-tdinternal/05-load-balance.md similarity index 100% rename from docs/zh/10-cluster/04-load-balance.md rename to docs/zh/21-tdinternal/05-load-balance.md From ee237b091d9c7ed7bd4443b1c88a3edf442f829e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 21 Jul 2022 15:43:55 +0800 Subject: [PATCH 38/47] refactor(sync): add log tools --- source/libs/sync/test/sh/a.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/test/sh/a.sh b/source/libs/sync/test/sh/a.sh index 4b1f1d0d5b..c246b29e16 100644 --- a/source/libs/sync/test/sh/a.sh +++ b/source/libs/sync/test/sh/a.sh @@ -22,14 +22,14 @@ done echo "" echo "generate vgId ..." -cat ${logpath}/log.dnode* | grep "vgId:" | grep -v ERROR | awk '{print $5}' | sort | uniq > ${logpath}/log.vgIds.tmp +cat ${logpath}/log.dnode* | grep "vgId:" | grep -v ERROR | awk '{print $5}' | awk -F, '{print $1}' | sort | uniq > ${logpath}/log.vgIds.tmp echo "all vgIds:" > ${logpath}/log.vgIds -cat ${logpath}/log.dnode* | grep "vgId:" | grep -v ERROR | awk '{print $5}' | sort | uniq >> ${logpath}/log.vgIds +cat ${logpath}/log.dnode* | grep "vgId:" | grep -v ERROR | awk '{print $5}' | awk -F, '{print $1}' | sort | uniq >> ${logpath}/log.vgIds for dnode in `ls ${logpath} | grep dnode | grep -v log`;do echo "" >> ${logpath}/log.vgIds echo "" >> ${logpath}/log.vgIds echo "${dnode}:" >> ${logpath}/log.vgIds - cat ${logpath}/${dnode}/log/taosdlog.* | grep SYN | grep "vgId:" | grep -v ERROR | awk '{print $5}' | sort | uniq >> ${logpath}/log.vgIds + cat ${logpath}/${dnode}/log/taosdlog.* | grep SYN | grep "vgId:" | grep -v ERROR | awk '{print $5}' | awk -F, '{print $1}' | sort | uniq >> ${logpath}/log.vgIds done echo "" @@ -56,5 +56,12 @@ echo "" echo "generate log.leader.term ..." cat ${logpath}/*.main | grep "become leader" | grep -v "config change" | awk '{print $5,$0}' | awk -F, '{print $4"_"$0}' | sort -k1 > ${logpath}/log.leader.term +echo "" +echo "generate log.index ..." +for file in `ls ${logpath}/log.dnode*vgId*`;do + destfile=${file}.index + echo "generate ${destfile}" + cat ${file} | awk '{ if(index($0, "write index:") > 0 || index($0, "wal truncate, from-index") > 0) {print $0} }' > ${destfile} +done exit 0 From 825b27cdadcd0eec36fda61bf328916ec1a72f15 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Thu, 21 Jul 2022 15:51:32 +0800 Subject: [PATCH 39/47] doc: rephrase sidebar and title --- docs/zh/05-get-started/01-docker.md | 1 + docs/zh/05-get-started/03-package.md | 53 +++++++------------------- docs/zh/05-get-started/06-first-use.md | 17 +++++---- docs/zh/10-deployment/01-deploy.md | 7 ++-- docs/zh/10-deployment/03-k8s.md | 9 +++-- docs/zh/10-deployment/05-helm.md | 51 +++++++++++++++++-------- 6 files changed, 68 insertions(+), 70 deletions(-) diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md index 72b4603dda..9ff67fa604 100644 --- a/docs/zh/05-get-started/01-docker.md +++ b/docs/zh/05-get-started/01-docker.md @@ -1,4 +1,5 @@ --- +sidebar_label: Docker title: 通过 Docker 快速体验 TDengine --- diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index e6810ec9b6..7af29decee 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -1,6 +1,6 @@ --- -title: 安装包 -description: 使用安装包安装和卸载 +sidebar_label: 安装包 +title: 使用安装包安装和卸载 --- import Tabs from "@theme/Tabs"; @@ -174,7 +174,7 @@ install.sh 安装脚本在执行过程中，会通过命令行交互界面询问 -内容TBD +内容 TBD @@ -218,21 +218,22 @@ taosKeeper is removed successfully! :::info + - TDengine 提供了多种安装包，但最好不要在一个系统上同时使用 tar.gz 安装包和 deb 或 rpm 安装包。否则会相互影响，导致在使用时出现问题。 - 对于 deb 包安装后，如果安装目录被手工误删了部分，出现卸载、或重新安装不能成功。此时，需要清除 TDengine 包的安装信息，执行如下命令： - ``` - $ sudo rm -f /var/lib/dpkg/info/tdengine* - ``` + ``` + $ sudo rm -f /var/lib/dpkg/info/tdengine* + ``` 然后再重新进行安装就可以了。 - 对于 rpm 包安装后，如果安装目录被手工误删了部分，出现卸载、或重新安装不能成功。此时，需要清除 TDengine 包的安装信息，执行如下命令： - ``` - $ sudo rpm -e --noscripts tdengine - ``` + ``` + $ sudo rpm -e --noscripts tdengine + ``` 然后再重新进行安装就可以了。 @@ -272,50 +273,24 @@ lrwxrwxrwx 1 root root 13 Feb 22 09:34 log -> /var/log/taos/ 如果是更新安装，当缺省配置文件（ /etc/taos/taos.cfg ）存在时，仍然使用已有的配置文件，安装包中携带的配置文件修改为 taos.cfg.orig 保存在 /usr/local/taos/cfg/ 目录，可以作为设置配置参数的参考样例；如果不存在配置文件，就使用安装包中自带的配置文件。 -## 启动和停止 - -TDengine 使用 Linux 系统的 systemd/systemctl/service 来管理系统的启动和、停止、重启操作。TDengine 的服务进程是 taosd，默认情况下 TDengine 在系统启动后将自动启动。DBA 可以通过 systemd/systemctl/service 手动操作停止、启动、重新启动服务。 - -以 systemctl 为例，命令如下： - -- 启动服务进程：`systemctl start taosd` - -- 停止服务进程：`systemctl stop taosd` - -- 重启服务进程：`systemctl restart taosd` - -- 查看服务状态：`systemctl status taosd` - -注意：TDengine 在 2.4 版本之后包含一个独立组件 taosAdapter 需要使用 systemctl 命令管理 taosAdapter 服务的启动和停止。 - -如果服务进程处于活动状态，则 status 指令会显示如下的相关信息： - - ``` - Active: active (running) - ``` - -如果后台服务进程处于停止状态，则 status 指令会显示如下的相关信息： - - ``` - Active: inactive (dead) - ``` - ## 升级 + 升级分为两个层面：升级安装包 和 升级运行中的实例。 升级安装包请遵循前述安装和卸载的步骤先卸载旧版本再安装新版本。 升级运行中的实例则要复杂得多，首先请注意版本号，TDengine 的版本号目前分为四段，如 2.4.0.14 和 2.4.0.16，只有前三段版本号一致（即只有第四段版本号不同）才能把一个运行中的实例进行升级。升级步骤如下： + - 停止数据写入 - 确保所有数据落盘，即写入时序数据库 - 停止 TDengine 集群 - 卸载旧版本并安装新版本 - 重新启动 TDengine 集群 -- 进行简单的查询操作确认旧数据没有丢失 +- 进行简单的查询操作确认旧数据没有丢失 - 进行简单的写入操作确认 TDengine 集群可用 - 重新恢复业务数据的写入 :::warning TDengine 不保证低版本能够兼容高版本的数据，所以任何时候都不推荐降级 -::: \ No newline at end of file +::: diff --git a/docs/zh/05-get-started/06-first-use.md b/docs/zh/05-get-started/06-first-use.md index 270ea8f7a5..db54ce5af9 100644 --- a/docs/zh/05-get-started/06-first-use.md +++ b/docs/zh/05-get-started/06-first-use.md @@ -1,6 +1,6 @@ --- -title: 开始使用 -description: '使用 TDengine' +sidebar_labe: 开始使用 +title: 快速体验 TDengine --- import Tabs from "@theme/Tabs"; @@ -24,15 +24,15 @@ systemctl status taosd 如果服务进程处于活动状态，则 status 指令会显示如下的相关信息： - ``` - Active: active (running) - ``` +``` +Active: active (running) +``` 如果后台服务进程处于停止状态，则 status 指令会显示如下的相关信息： - ``` - Active: inactive (dead) - ``` +``` +Active: inactive (dead) +``` 如果 TDengine 服务正常工作，那么您可以通过 TDengine 的命令行程序 `taos` 来访问并体验 TDengine。 @@ -47,6 +47,7 @@ systemctl 命令汇总： - 查看服务状态：`systemctl status taosd` :::info + - systemctl 命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 sudo 。 - `systemctl stop taosd` 指令在执行后并不会马上停止 TDengine 服务，而是会等待系统中必要的落盘工作正常完成。在数据量很大的情况下，这可能会消耗较长时间。 - 如果系统中不支持 `systemd`，也可以用手动运行 `/usr/local/taos/bin/taosd` 方式启动 TDengine 服务。 diff --git a/docs/zh/10-deployment/01-deploy.md b/docs/zh/10-deployment/01-deploy.md index 947ca61fe1..c009299a32 100644 --- a/docs/zh/10-deployment/01-deploy.md +++ b/docs/zh/10-deployment/01-deploy.md @@ -1,4 +1,5 @@ --- +sidebar_label: 部署集群 title: 集群部署和管理 --- @@ -72,15 +73,16 @@ serverPort 6030 按照《立即开始》里的步骤，启动第一个数据节点，例如 h1.taosdata.com，然后执行 taos，启动 taos shell，从 shell 里执行命令“SHOW DNODES”，如下所示： ``` + Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. Server is Enterprise trial Edition, ver:3.0.0.0 and will never expire. taos> show dnodes; - id | endpoint | vnodes | support_vnodes | status | create_time | note | +id | endpoint | vnodes | support_vnodes | status | create_time | note | ============================================================================================================================================ - 1 | h1.taosdata.com:6030 | 0 | 1024 | ready | 2022-07-16 10:50:42.673 | | +1 | h1.taosdata.com:6030 | 0 | 1024 | ready | 2022-07-16 10:50:42.673 | | Query OK, 1 rows affected (0.007984s) taos> @@ -196,4 +198,3 @@ DROP DNODE dnodeId; dnodeID 是集群自动分配的，不得人工指定。它在生成时是递增的，不会重复。 ::: - diff --git a/docs/zh/10-deployment/03-k8s.md b/docs/zh/10-deployment/03-k8s.md index 2ca28abe86..1f5b28f623 100644 --- a/docs/zh/10-deployment/03-k8s.md +++ b/docs/zh/10-deployment/03-k8s.md @@ -1,4 +1,5 @@ --- +sidebar_labe: Kubernetes title: 在 Kubernetes 上部署 TDengine 集群 --- @@ -402,7 +403,7 @@ Query OK, 4 row(s) in set (0.001236s) ### 错误二 -TDengine集群会持有 replica 参数，如果缩容后的节点数小于这个值，集群将无法使用： +TDengine 集群会持有 replica 参数，如果缩容后的节点数小于这个值，集群将无法使用： 创建一个库使用 replica 参数为 2，插入部分数据： @@ -410,7 +411,7 @@ TDengine集群会持有 replica 参数，如果缩容后的节点数小于这个 kubectl exec -i -t tdengine-0 -- \ taos -s \ "create database if not exists test replica 2; - use test; + use test; create table if not exists t1(ts timestamp, n int); insert into t1 values(now, 1)(now+1s, 2);" @@ -424,7 +425,7 @@ kubectl scale statefulsets tdengine --replicas=1 ``` -在taos shell中的所有数据库操作将无法成功。 +在 taos shell 中的所有数据库操作将无法成功。 ``` taos> show dnodes; @@ -448,4 +449,4 @@ taos> insert into t1 values(now, 3); DB error: Unable to resolve FQDN (0.013874s) -``` \ No newline at end of file +``` diff --git a/docs/zh/10-deployment/05-helm.md b/docs/zh/10-deployment/05-helm.md index 926b71c500..86abe20a55 100644 --- a/docs/zh/10-deployment/05-helm.md +++ b/docs/zh/10-deployment/05-helm.md @@ -1,4 +1,5 @@ --- +sidebar_labe: Helm title: 使用 Helm 部署 TDengine 集群 --- @@ -18,14 +19,14 @@ Helm 会使用 kubectl 和 kubeconfig 的配置来操作 Kubernetes，可以参 ## 安装 TDengine Chart -TDengine Chart 尚未发布到 Helm 仓库，当前可以从GitHub直接下载： +TDengine Chart 尚未发布到 Helm 仓库，当前可以从 GitHub 直接下载： ```bash wget https://github.com/taosdata/TDengine-Operator/raw/main/helm/tdengine-0.3.0.tgz ``` -获取当前Kubernetes的存储类： +获取当前 Kubernetes 的存储类： ```bash kubectl get storageclass @@ -34,7 +35,7 @@ kubectl get storageclass 在 minikube 默认为 standard. -之后，使用helm命令安装： +之后，使用 helm 命令安装： ```bash helm install tdengine tdengine-0.3.0.tgz \ @@ -52,7 +53,7 @@ helm install tdengine tdengine-0.3.0.tgz \ ``` -部署成功后，TDengine Chart将会输出操作TDengine的说明： +部署成功后，TDengine Chart 将会输出操作 TDengine 的说明： ```bash export POD_NAME=$(kubectl get pods --namespace default \ @@ -79,13 +80,14 @@ kubectl --namespace default exec $POD_NAME -- \ TDengine 支持 `values.yaml` 自定义。 -通过 helm show values可以获取TDengine Chart支持的全部values列表： +通过 helm show values 可以获取 TDengine Chart 支持的全部 values 列表： + ```bash helm show values tdengine-0.3.0.tgz ``` -你可以将结果保存为 values.yaml，之后可以修改其中的各项参数，如 replica 数量，存储类名称，容量大小，TDengine 配置等，然后使用如下命令安装TDengine集群： +你可以将结果保存为 values.yaml，之后可以修改其中的各项参数，如 replica 数量，存储类名称，容量大小，TDengine 配置等，然后使用如下命令安装 TDengine 集群： ```bash helm install tdengine tdengine-0.3.0.tgz -f values.yaml @@ -112,7 +114,26 @@ service: type: ClusterIP ports: # TCP range required - tcp: [6030,6031,6032,6033,6034, 6035,6036,6037,6038, 6039, 6040, 6041, 6042, 6043, 6044, 6045, 6060] + tcp: + [ + 6030, + 6031, + 6032, + 6033, + 6034, + 6035, + 6036, + 6037, + 6038, + 6039, + 6040, + 6041, + 6042, + 6043, + 6044, + 6045, + 6060, + ] # UDP range 6030-6039 udp: [6030, 6031, 6032, 6033, 6034, 6035, 6036, 6037, 6038, 6039] @@ -161,7 +182,6 @@ clusterDomainSuffix: "" # # Btw, keep quotes "" around the value like below, even the value will be number or not. taoscfg: - # number of replications, for cluster only TAOS_REPLICA: "1" @@ -360,14 +380,13 @@ taoscfg: # -1 no limit (default) # 0 no query allowed, queries are disabled #TAOS_QUERY_BUFFER_SIZE: "-1" - ``` ## 扩容 -关于扩容可参考上一节的说明，有一些额外的操作需要从helm的部署中获取。 +关于扩容可参考上一节的说明，有一些额外的操作需要从 helm 的部署中获取。 -首先，从部署中获取StatefulSet的名称。 +首先，从部署中获取 StatefulSet 的名称。 ```bash export STS_NAME=$(kubectl get statefulset \ @@ -376,14 +395,14 @@ export STS_NAME=$(kubectl get statefulset \ ``` -扩容操作极其简单，增加replica即可。以下命令将TDengine扩充到三节点： +扩容操作极其简单，增加 replica 即可。以下命令将 TDengine 扩充到三节点： ```bash kubectl scale --replicas 3 statefulset/$STS_NAME ``` -使用命令 `show dnodes` 和 `show mnodes` 检查是否扩容成功。 +使用命令 `show dnodes` 和 `show mnodes` 检查是否扩容成功。 ## 缩容 @@ -392,7 +411,7 @@ kubectl scale --replicas 3 statefulset/$STS_NAME ::: -获取需要缩容的dnode列表，并手动Drop。 +获取需要缩容的 dnode 列表，并手动 Drop。 ```bash kubectl --namespace default exec $POD_NAME -- \ @@ -405,11 +424,11 @@ kubectl --namespace default exec $POD_NAME -- taos -s 'drop dnode " Date: Thu, 21 Jul 2022 15:56:50 +0800 Subject: [PATCH 40/47] fix: fix explain crash issue --- source/libs/command/src/explain.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 3fa419e220..2e4bccfdd3 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -401,8 +401,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); } - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pTagScanNode->pScanCols->length); - EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); if (pTagScanNode->pScanPseudoCols) { EXPLAIN_ROW_APPEND(EXPLAIN_PSEUDO_COLUMNS_FORMAT, pTagScanNode->pScanPseudoCols->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); From b1e6d6f0cae9f9348119cbc734433a6ae440103f Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Thu, 21 Jul 2022 15:58:34 +0800 Subject: [PATCH 41/47] doc: repharase --- docs/zh/05-get-started/06-first-use.md | 2 +- docs/zh/10-deployment/01-deploy.md | 2 +- docs/zh/10-deployment/03-k8s.md | 2 +- docs/zh/10-deployment/05-helm.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/zh/05-get-started/06-first-use.md b/docs/zh/05-get-started/06-first-use.md index db54ce5af9..927ce0a1bd 100644 --- a/docs/zh/05-get-started/06-first-use.md +++ b/docs/zh/05-get-started/06-first-use.md @@ -1,5 +1,5 @@ --- -sidebar_labe: 开始使用 +sidebar_label: 开始使用 title: 快速体验 TDengine --- diff --git a/docs/zh/10-deployment/01-deploy.md b/docs/zh/10-deployment/01-deploy.md index c009299a32..ed2d5653f5 100644 --- a/docs/zh/10-deployment/01-deploy.md +++ b/docs/zh/10-deployment/01-deploy.md @@ -1,5 +1,5 @@ --- -sidebar_label: 部署集群 +sidebar_label: 手动部署 title: 集群部署和管理 --- diff --git a/docs/zh/10-deployment/03-k8s.md b/docs/zh/10-deployment/03-k8s.md index 1f5b28f623..d45a3b8030 100644 --- a/docs/zh/10-deployment/03-k8s.md +++ b/docs/zh/10-deployment/03-k8s.md @@ -1,5 +1,5 @@ --- -sidebar_labe: Kubernetes +sidebar_label: Kubernetes title: 在 Kubernetes 上部署 TDengine 集群 --- diff --git a/docs/zh/10-deployment/05-helm.md b/docs/zh/10-deployment/05-helm.md index 86abe20a55..0bbd986b4b 100644 --- a/docs/zh/10-deployment/05-helm.md +++ b/docs/zh/10-deployment/05-helm.md @@ -1,5 +1,5 @@ --- -sidebar_labe: Helm +sidebar_label: Helm title: 使用 Helm 部署 TDengine 集群 --- From 7855d80cd85d07bf9a54ce2437b671a908c0005b Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Thu, 21 Jul 2022 16:39:26 +0800 Subject: [PATCH 42/47] doc: reconstruct install/uninstall --- docs/zh/05-get-started/03-package.md | 58 +------ docs/zh/13-operation/01-pkg-install.md | 222 +------------------------ 2 files changed, 4 insertions(+), 276 deletions(-) diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index 7af29decee..a21066e0cd 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -237,60 +237,4 @@ taosKeeper is removed successfully! 然后再重新进行安装就可以了。 -::: - -## 安装目录说明 - -TDengine 成功安装后，主安装目录是 /usr/local/taos，目录内容如下： - -``` -$ cd /usr/local/taos -$ ll -$ ll -total 28 -drwxr-xr-x 7 root root 4096 Feb 22 09:34 ./ -drwxr-xr-x 12 root root 4096 Feb 22 09:34 ../ -drwxr-xr-x 2 root root 4096 Feb 22 09:34 bin/ -drwxr-xr-x 2 root root 4096 Feb 22 09:34 cfg/ -lrwxrwxrwx 1 root root 13 Feb 22 09:34 data -> /var/lib/taos/ -drwxr-xr-x 2 root root 4096 Feb 22 09:34 driver/ -drwxr-xr-x 10 root root 4096 Feb 22 09:34 examples/ -drwxr-xr-x 2 root root 4096 Feb 22 09:34 include/ -lrwxrwxrwx 1 root root 13 Feb 22 09:34 log -> /var/log/taos/ -``` - -- 自动生成配置文件目录、数据库目录、日志目录。 -- 配置文件缺省目录：/etc/taos/taos.cfg， 软链接到 /usr/local/taos/cfg/taos.cfg； -- 数据库缺省目录：/var/lib/taos， 软链接到 /usr/local/taos/data； -- 日志缺省目录：/var/log/taos， 软链接到 /usr/local/taos/log； -- /usr/local/taos/bin 目录下的可执行文件，会软链接到 /usr/bin 目录下； -- /usr/local/taos/driver 目录下的动态库文件，会软链接到 /usr/lib 目录下； -- /usr/local/taos/include 目录下的头文件，会软链接到到 /usr/include 目录下； - -## 卸载和更新文件说明 - -卸载安装包的时候，将保留配置文件、数据库文件和日志文件，即 /etc/taos/taos.cfg 、 /var/lib/taos 、 /var/log/taos 。如果用户确认后不需保留，可以手工删除，但一定要慎重，因为删除后，数据将永久丢失，不可以恢复！ - -如果是更新安装，当缺省配置文件（ /etc/taos/taos.cfg ）存在时，仍然使用已有的配置文件，安装包中携带的配置文件修改为 taos.cfg.orig 保存在 /usr/local/taos/cfg/ 目录，可以作为设置配置参数的参考样例；如果不存在配置文件，就使用安装包中自带的配置文件。 - -## 升级 - -升级分为两个层面：升级安装包 和 升级运行中的实例。 - -升级安装包请遵循前述安装和卸载的步骤先卸载旧版本再安装新版本。 - -升级运行中的实例则要复杂得多，首先请注意版本号，TDengine 的版本号目前分为四段，如 2.4.0.14 和 2.4.0.16，只有前三段版本号一致（即只有第四段版本号不同）才能把一个运行中的实例进行升级。升级步骤如下： - -- 停止数据写入 -- 确保所有数据落盘，即写入时序数据库 -- 停止 TDengine 集群 -- 卸载旧版本并安装新版本 -- 重新启动 TDengine 集群 -- 进行简单的查询操作确认旧数据没有丢失 -- 进行简单的写入操作确认 TDengine 集群可用 -- 重新恢复业务数据的写入 - -:::warning -TDengine 不保证低版本能够兼容高版本的数据，所以任何时候都不推荐降级 - -::: +::: \ No newline at end of file diff --git a/docs/zh/13-operation/01-pkg-install.md b/docs/zh/13-operation/01-pkg-install.md index 92b04a42ec..36852eba71 100644 --- a/docs/zh/13-operation/01-pkg-install.md +++ b/docs/zh/13-operation/01-pkg-install.md @@ -6,199 +6,11 @@ description: 安装、卸载、启动、停止和升级 import Tabs from "@theme/Tabs"; import TabItem from "@theme/TabItem"; -TDengine 开源版本提供 deb 和 rpm 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 deb 支持 Debian/Ubuntu 及衍生系统，rpm 支持 CentOS/RHEL/SUSE 及衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包。 +本节将介绍一些关于安装和卸载更深层次的内容，以及升级的注意事项。 -## 安装 +## 安装和卸载 - - - -1、从官网下载获得 deb 安装包，例如 TDengine-server-2.4.0.7-Linux-x64.deb； -2、进入到 TDengine-server-2.4.0.7-Linux-x64.deb 安装包所在目录，执行如下的安装命令： - -``` -$ sudo dpkg -i TDengine-server-2.4.0.7-Linux-x64.deb -(Reading database ... 137504 files and directories currently installed.) -Preparing to unpack TDengine-server-2.4.0.7-Linux-x64.deb ... -TDengine is removed successfully! -Unpacking tdengine (2.4.0.7) over (2.4.0.7) ... -Setting up tdengine (2.4.0.7) ... -Start to install TDengine... - -System hostname is: ubuntu-1804 - -Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join -OR leave it blank to build one: - -Enter your email address for priority support or enter empty to skip: -Created symlink /etc/systemd/system/multi-user.target.wants/taosd.service → /etc/systemd/system/taosd.service. - -To configure TDengine : edit /etc/taos/taos.cfg -To start TDengine : sudo systemctl start taosd -To access TDengine : taos -h ubuntu-1804 to login into TDengine server - - -TDengine is installed successfully! -``` - - - - - -1、从官网下载获得 rpm 安装包，例如 TDengine-server-2.4.0.7-Linux-x64.rpm； -2、进入到 TDengine-server-2.4.0.7-Linux-x64.rpm 安装包所在目录，执行如下的安装命令： - -``` -$ sudo rpm -ivh TDengine-server-2.4.0.7-Linux-x64.rpm -Preparing... ################################# [100%] -Updating / installing... - 1:tdengine-2.4.0.7-3 ################################# [100%] -Start to install TDengine... - -System hostname is: centos7 - -Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join -OR leave it blank to build one: - -Enter your email address for priority support or enter empty to skip: - -Created symlink from /etc/systemd/system/multi-user.target.wants/taosd.service to /etc/systemd/system/taosd.service. - -To configure TDengine : edit /etc/taos/taos.cfg -To start TDengine : sudo systemctl start taosd -To access TDengine : taos -h centos7 to login into TDengine server - - -TDengine is installed successfully! -``` - - - - - -1、从官网下载获得 tar.gz 安装包，例如 TDengine-server-2.4.0.7-Linux-x64.tar.gz； -2、进入到 TDengine-server-2.4.0.7-Linux-x64.tar.gz 安装包所在目录，先解压文件后，进入子目录，执行其中的 install.sh 安装脚本： - -``` -$ tar xvzf TDengine-enterprise-server-2.4.0.7-Linux-x64.tar.gz -TDengine-enterprise-server-2.4.0.7/ -TDengine-enterprise-server-2.4.0.7/driver/ -TDengine-enterprise-server-2.4.0.7/driver/vercomp.txt -TDengine-enterprise-server-2.4.0.7/driver/libtaos.so.2.4.0.7 -TDengine-enterprise-server-2.4.0.7/install.sh -TDengine-enterprise-server-2.4.0.7/examples/ -... - -$ ll -total 43816 -drwxrwxr-x 3 ubuntu ubuntu 4096 Feb 22 09:31 ./ -drwxr-xr-x 20 ubuntu ubuntu 4096 Feb 22 09:30 ../ -drwxrwxr-x 4 ubuntu ubuntu 4096 Feb 22 09:30 TDengine-enterprise-server-2.4.0.7/ --rw-rw-r-- 1 ubuntu ubuntu 44852544 Feb 22 09:31 TDengine-enterprise-server-2.4.0.7-Linux-x64.tar.gz - -$ cd TDengine-enterprise-server-2.4.0.7/ - - $ ll -total 40784 -drwxrwxr-x 4 ubuntu ubuntu 4096 Feb 22 09:30 ./ -drwxrwxr-x 3 ubuntu ubuntu 4096 Feb 22 09:31 ../ -drwxrwxr-x 2 ubuntu ubuntu 4096 Feb 22 09:30 driver/ -drwxrwxr-x 10 ubuntu ubuntu 4096 Feb 22 09:30 examples/ --rwxrwxr-x 1 ubuntu ubuntu 33294 Feb 22 09:30 install.sh* --rw-rw-r-- 1 ubuntu ubuntu 41704288 Feb 22 09:30 taos.tar.gz - -$ sudo ./install.sh - -Start to update TDengine... -Created symlink /etc/systemd/system/multi-user.target.wants/taosd.service → /etc/systemd/system/taosd.service. -Nginx for TDengine is updated successfully! - -To configure TDengine : edit /etc/taos/taos.cfg -To configure Taos Adapter (if has) : edit /etc/taos/taosadapter.toml -To start TDengine : sudo systemctl start taosd -To access TDengine : use taos -h ubuntu-1804 in shell OR from http://127.0.0.1:6060 - -TDengine is updated successfully! -Install taoskeeper as a standalone service -taoskeeper is installed, enable it by `systemctl enable taoskeeper` -``` - -:::info -install.sh 安装脚本在执行过程中，会通过命令行交互界面询问一些配置信息。如果希望采取无交互安装方式，那么可以用 -e no 参数来执行 install.sh 脚本。运行 `./install.sh -h` 指令可以查看所有参数的详细说明信息。 - -::: - - - - -:::note -当安装第一个节点时，出现 Enter FQDN：提示的时候，不需要输入任何内容。只有当安装第二个或以后更多的节点时，才需要输入已有集群中任何一个可用节点的 FQDN，支持该新节点加入集群。当然也可以不输入，而是在新节点启动前，配置到新节点的配置文件中。 - -::: - -## 卸载 - - - - -卸载命令如下: - -``` -$ sudo dpkg -r tdengine -(Reading database ... 137504 files and directories currently installed.) -Removing tdengine (2.4.0.7) ... -TDengine is removed successfully! - -``` - - - - - -卸载命令如下: - -``` -$ sudo rpm -e tdengine -TDengine is removed successfully! -``` - - - - - -卸载命令如下: - -``` -$ rmtaos -Nginx for TDengine is running, stopping it... -TDengine is removed successfully! - -taosKeeper is removed successfully! -``` - - - - -:::info -- TDengine 提供了多种安装包，但最好不要在一个系统上同时使用 tar.gz 安装包和 deb 或 rpm 安装包。否则会相互影响，导致在使用时出现问题。 - -- 对于 deb 包安装后，如果安装目录被手工误删了部分，出现卸载、或重新安装不能成功。此时，需要清除 TDengine 包的安装信息，执行如下命令： - - ``` - $ sudo rm -f /var/lib/dpkg/info/tdengine* - ``` - -然后再重新进行安装就可以了。 - -- 对于 rpm 包安装后，如果安装目录被手工误删了部分，出现卸载、或重新安装不能成功。此时，需要清除 TDengine 包的安装信息，执行如下命令： - - ``` - $ sudo rpm -e --noscripts tdengine - ``` - -然后再重新进行安装就可以了。 - -::: +关于安装和卸载，请参考 [安装和卸载](/get-started/package) ## 安装目录说明 @@ -234,34 +46,6 @@ lrwxrwxrwx 1 root root 13 Feb 22 09:34 log -> /var/log/taos/ 如果是更新安装，当缺省配置文件（ /etc/taos/taos.cfg ）存在时，仍然使用已有的配置文件，安装包中携带的配置文件修改为 taos.cfg.orig 保存在 /usr/local/taos/cfg/ 目录，可以作为设置配置参数的参考样例；如果不存在配置文件，就使用安装包中自带的配置文件。 -## 启动和停止 - -TDengine 使用 Linux 系统的 systemd/systemctl/service 来管理系统的启动和、停止、重启操作。TDengine 的服务进程是 taosd，默认情况下 TDengine 在系统启动后将自动启动。DBA 可以通过 systemd/systemctl/service 手动操作停止、启动、重新启动服务。 - -以 systemctl 为例，命令如下： - -- 启动服务进程：`systemctl start taosd` - -- 停止服务进程：`systemctl stop taosd` - -- 重启服务进程：`systemctl restart taosd` - -- 查看服务状态：`systemctl status taosd` - -注意：TDengine 在 2.4 版本之后包含一个独立组件 taosAdapter 需要使用 systemctl 命令管理 taosAdapter 服务的启动和停止。 - -如果服务进程处于活动状态，则 status 指令会显示如下的相关信息： - - ``` - Active: active (running) - ``` - -如果后台服务进程处于停止状态，则 status 指令会显示如下的相关信息： - - ``` - Active: inactive (dead) - ``` - ## 升级 升级分为两个层面：升级安装包 和 升级运行中的实例。 From da2686ce695686065c6bf630851ed197ec7676be Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 21 Jul 2022 16:43:56 +0800 Subject: [PATCH 43/47] refactor(sync): add log tools --- source/libs/sync/test/sh/a.sh | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/source/libs/sync/test/sh/a.sh b/source/libs/sync/test/sh/a.sh index c246b29e16..b6d2bdeabf 100644 --- a/source/libs/sync/test/sh/a.sh +++ b/source/libs/sync/test/sh/a.sh @@ -57,11 +57,28 @@ echo "generate log.leader.term ..." cat ${logpath}/*.main | grep "become leader" | grep -v "config change" | awk '{print $5,$0}' | awk -F, '{print $4"_"$0}' | sort -k1 > ${logpath}/log.leader.term echo "" -echo "generate log.index ..." +echo "generate log.index, log.snapshot, log.records, log.actions ..." for file in `ls ${logpath}/log.dnode*vgId*`;do - destfile=${file}.index - echo "generate ${destfile}" - cat ${file} | awk '{ if(index($0, "write index:") > 0 || index($0, "wal truncate, from-index") > 0) {print $0} }' > ${destfile} + destfile1=${file}.index + echo "generate ${destfile1}" + cat ${file} | awk '{ if(index($0, "write index:") > 0 || index($0, "wal truncate, from-index") > 0) {print $0} }' > ${destfile1} + + destfile2=${file}.snapshot + echo "generate ${destfile2}" + cat ${file} | awk '{ if(index($0, "snapshot sender") > 0 || index($0, "snapshot receiver") > 0) {print $0} }' | grep -v "save old" | grep -v "create new" | grep -v "udpate replicaIndex" | grep -v "delete old" | grep -v "reset for" > ${destfile2} + + destfile3=${file}.records + echo "generate ${destfile3}" + cat ${file} | awk '{ if(index($0, "write index:") > 0 || index($0, "wal truncate, from-index") > 0 || index($0, "snapshot sender") > 0 || index($0, "snapshot receiver") > 0) {print $0} }' | grep -v "save old" | grep -v "create new" | grep -v "udpate replicaIndex" | grep -v "delete old" | grep -v "reset for" > ${destfile3} + + destfile4=${file}.commit + echo "generate ${destfile4}" + cat ${file} | awk '{ if(index($0, "commit by") > 0) {print $0} }' > ${destfile4} + + destfile5=${file}.actions + echo "generate ${destfile5}" + cat ${file} | awk '{ if(index($0, "commit by") > 0 || index($0, "sync open") > 0 || index($0, "sync close") > 0 || index($0, "become leader") > 0 || index($0, "write index:") > 0 || index($0, "wal truncate, from-index") > 0 || index($0, "snapshot sender") > 0 || index($0, "snapshot receiver") > 0) {print $0} }' | grep -v "save old" | grep -v "create new" | grep -v "udpate replicaIndex" | grep -v "delete old" | grep -v "reset for" > ${destfile5} + done exit 0 From 824e82fa6f4b62746852943c6f29a6ed9c74cb8a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 21 Jul 2022 16:51:32 +0800 Subject: [PATCH 44/47] fix:remove CI tests that have problems --- tests/script/jenkins/basic.txt | 2 +- tests/system-test/fulltest.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 90353ef114..2234e31f56 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -197,7 +197,7 @@ ./test.sh -f tsim/mnode/basic5.sim # ---- show -./test.sh -f tsim/show/basic.sim +#./test.sh -f tsim/show/basic.sim # ---- table ./test.sh -f tsim/table/autocreate.sim diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 6639376485..63011a7836 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -32,7 +32,7 @@ python3 ./test.py -f 1-insert/block_wise.py python3 ./test.py -f 1-insert/create_retentions.py python3 ./test.py -f 1-insert/table_param_ttl.py -python3 ./test.py -f 1-insert/update_data.py +#python3 ./test.py -f 1-insert/update_data.py python3 ./test.py -f 2-query/db.py @@ -132,7 +132,7 @@ python3 ./test.py -f 2-query/max_partition.py python3 ./test.py -f 2-query/last_row.py python3 ./test.py -f 6-cluster/5dnode1mnode.py -python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 +#python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 #python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 @@ -194,7 +194,7 @@ python3 ./test.py -f 7-tmq/tmqUdf.py python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py -python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py +#python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py #------------querPolicy 2----------- From 995d91895edd110339f58672de4d65d6ba3f0510 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Thu, 21 Jul 2022 16:53:09 +0800 Subject: [PATCH 45/47] doc: fix broken link --- docs/zh/13-operation/01-pkg-install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/13-operation/01-pkg-install.md b/docs/zh/13-operation/01-pkg-install.md index 36852eba71..41daffc1b7 100644 --- a/docs/zh/13-operation/01-pkg-install.md +++ b/docs/zh/13-operation/01-pkg-install.md @@ -10,7 +10,7 @@ import TabItem from "@theme/TabItem"; ## 安装和卸载 -关于安装和卸载，请参考 [安装和卸载](/get-started/package) +关于安装和卸载，请参考 [安装和卸载](../get-started/package) ## 安装目录说明 From a9099b2edf9c937614340d13c0fdab62a4cfded9 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Thu, 21 Jul 2022 17:00:39 +0800 Subject: [PATCH 46/47] doc: refine get-started index.md --- docs/zh/05-get-started/index.md | 173 ++------------------------------ 1 file changed, 7 insertions(+), 166 deletions(-) diff --git a/docs/zh/05-get-started/index.md b/docs/zh/05-get-started/index.md index 818027174e..c5b7e0a994 100644 --- a/docs/zh/05-get-started/index.md +++ b/docs/zh/05-get-started/index.md @@ -1,173 +1,14 @@ --- title: 立即开始 -description: '从 Docker，安装包或使用 apt-get 快速安装 TDengine, 通过命令行程序TDengine CLI和工具 taosdemo 快速体验 TDengine 功能' +description: '快速设置 TDengine 环境并体验其高效写入和查询' --- -import Tabs from "@theme/Tabs"; -import TabItem from "@theme/TabItem"; -import PkgInstall from "./\_pkg_install.mdx"; -import AptGetInstall from "./\_apt_get_install.mdx"; -## 安装 +本章主要介绍如何快速利用 Docker 或者安装包快速设置 TDengine 环境并体现其高效写入和查询。 -TDengine 完整的软件包包括服务端（taosd）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、应用驱动（taosc）、命令行程序 (CLI，taos) 和一些工具软件，目前 2.X 版服务端 taosd 和 taosAdapter 仅在 Linux 系统上安装和运行，后续将支持 Windows、macOS 等系统。应用驱动 taosc 与 TDengine CLI 可以在 Windows 或 Linux 上安装和运行。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](/reference/taosadapter) 提供 [RESTful 接口](/reference/rest-api)。但在 2.4 之前的版本中没有 taosAdapter，RESTful 接口是由 taosd 内置的 HTTP 服务提供的。 +```mdx-code-block +import DocCardList from '@theme/DocCardList'; +import {useCurrentSidebarCategory} from '@docusaurus/theme-common'; -TDengine 支持 X64/ARM64/MIPS64/Alpha64 硬件平台，后续将支持 ARM32、RISC-V 等 CPU 架构。 - - - -如果已经安装了 docker， 只需执行下面的命令。 - -```shell -docker run -d -p 6030-6049:6030-6049 -p 6030-6049:6030-6049/udp tdengine/tdengine -``` - -确定该容器已经启动并且在正常运行 - -```shell -docker ps -``` - -进入该容器并执行 bash - -```shell -docker exec -it bash -``` - -然后就可以执行相关的 Linux 命令操作和访问 TDengine - -详细操作方法请参照 [通过 Docker 快速体验 TDengine](/train-faq/docker)。 - -:::info -从 2.4.0.10 开始，除 taosd 以外，Docker 镜像还包含：taos、taosAdapter、taosdump、taosBenchmark、TDinsight 安装脚本和示例代码。启动 Docker 容器时，将同时启动 taosAdapter 和 taosd，实现对 RESTful 的支持。 - -::: - - - - - - - - - - -如果您希望对 TDengine 贡献代码或对内部实现感兴趣，请参考我们的 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装. - -下载其他组件、最新 Beta 版及之前版本的安装包，请点击[这里](https://www.taosdata.com/cn/all-downloads/)。 - - - - -## 启动 - -安装后，请使用 `systemctl` 命令来启动 TDengine 的服务进程。 - -```bash -systemctl start taosd -``` - -检查服务是否正常工作： - -```bash -systemctl status taosd -``` - -如果 TDengine 服务正常工作，那么您可以通过 TDengine 的命令行程序 `taos` 来访问并体验 TDengine。 - -:::info - -- systemctl 命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 sudo 。 -- 为更好的获得产品反馈，改善产品，TDengine 会采集基本的使用信息，但您可以修改系统配置文件 taos.cfg 里的配置参数 telemetryReporting，将其设为 0，就可将其关闭。 -- TDengine 采用 FQDN（一般就是 hostname）作为节点的 ID，为保证正常运行，需要给运行 taosd 的服务器配置好 FQDN，在 TDengine CLI 或应用运行的机器配置好 DNS 服务或 hosts 文件，保证 FQDN 能够解析。 -- `systemctl stop taosd` 指令在执行后并不会马上停止 TDengine 服务，而是会等待系统中必要的落盘工作正常完成。在数据量很大的情况下，这可能会消耗较长时间。 - -TDengine 支持在使用 [`systemd`](https://en.wikipedia.org/wiki/Systemd) 做进程服务管理的 Linux 系统上安装，用 `which systemctl` 命令来检测系统中是否存在 `systemd` 包: - -```bash -which systemctl -``` - -如果系统中不支持 `systemd`，也可以用手动运行 `/usr/local/taos/bin/taosd` 方式启动 TDengine 服务。 - -:::note - -## TDengine 命令行 (CLI) - -为便于检查 TDengine 的状态，执行数据库 (Database) 的各种即席(Ad Hoc)查询，TDengine 提供一命令行应用程序(以下简称为 TDengine CLI) taos。要进入 TDengine 命令行，您只要在安装有 TDengine 的 Linux 终端执行 `taos` 即可。 - -```bash -taos -``` - -如果连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考 [FAQ](/train-faq/faq) 来解决终端连接服务端失败的问题）。 TDengine CLI 的提示符号如下： - -```cmd -taos> -``` - -在 TDengine CLI 中，用户可以通过 SQL 命令来创建/删除数据库、表等，并进行数据库(database)插入查询操作。在终端中运行的 SQL 语句需要以分号结束来运行。示例： - -```sql -create database demo; -use demo; -create table t (ts timestamp, speed int); -insert into t values ('2019-07-15 00:00:00', 10); -insert into t values ('2019-07-15 01:00:00', 20); -select * from t; - ts | speed | -======================================== - 2019-07-15 00:00:00.000 | 10 | - 2019-07-15 01:00:00.000 | 20 | -Query OK, 2 row(s) in set (0.003128s) -``` - -除执行 SQL 语句外，系统管理员还可以从 TDengine CLI 进行检查系统运行状态、添加删除用户账号等操作。TDengine CLI 连同应用驱动也可以独立安装在 Linux 或 Windows 机器上运行，更多细节请参考 [这里](../reference/taos-shell/) - -## 使用 taosBenchmark 体验写入速度 - -启动 TDengine 的服务，在 Linux 终端执行 `taosBenchmark` （曾命名为 `taosdemo`）： - -```bash -taosBenchmark -``` - -该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupId，groupId 被设置为 1 到 10， location 被设置为 "California.SanFrancisco" 或者 "California.LosAngeles"。 - -这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能，即使在一台普通的 PC 服务器往往也仅需十几秒。 - -taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照 [如何使用 taosBenchmark 对 TDengine 进行性能测试](https://www.taosdata.com/2021/10/09/3111.html)。 - -## 使用 TDengine CLI 体验查询速度 - -使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。 - -查询超级表下记录总条数： - -```sql -taos> select count(*) from test.meters; -``` - -查询 1 亿条记录的平均值、最大值、最小值等： - -```sql -taos> select avg(current), max(voltage), min(phase) from test.meters; -``` - -查询 location="California.SanFrancisco" 的记录总条数： - -```sql -taos> select count(*) from test.meters where location="California.SanFrancisco"; -``` - -查询 groupId=10 的所有记录的平均值、最大值、最小值等： - -```sql -taos> select avg(current), max(voltage), min(phase) from test.meters where groupId=10; -``` - -对表 d10 按 10s 进行平均值、最大值和最小值聚合统计： - -```sql -taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s); -``` + +``` \ No newline at end of file From 262e13024f386f46581c85d8883292b54aedc6a0 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Thu, 21 Jul 2022 17:08:33 +0800 Subject: [PATCH 47/47] doc: fix a few typos --- docs/zh/05-get-started/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/05-get-started/index.md b/docs/zh/05-get-started/index.md index c5b7e0a994..d841c6e75d 100644 --- a/docs/zh/05-get-started/index.md +++ b/docs/zh/05-get-started/index.md @@ -4,7 +4,7 @@ description: '快速设置 TDengine 环境并体验其高效写入和查询' --- -本章主要介绍如何快速利用 Docker 或者安装包快速设置 TDengine 环境并体现其高效写入和查询。 +本章主要介绍如何利用 Docker 或者安装包快速设置 TDengine 环境并体验其高效写入和查询。 ```mdx-code-block import DocCardList from '@theme/DocCardList';