From d4b72613fd7d2a4dd5ee3597bacbb474e85aca34 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Wed, 1 Jun 2022 17:57:30 +0800 Subject: [PATCH 01/26] test: add debug cases --- tests/pytest/insert/insert_drop.py | 49 ++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tests/pytest/insert/insert_drop.py diff --git a/tests/pytest/insert/insert_drop.py b/tests/pytest/insert/insert_drop.py new file mode 100644 index 0000000000..5ccaa5540b --- /dev/null +++ b/tests/pytest/insert/insert_drop.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +import threading + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def genMultiThreadSeq(self, sql_list): + tlist = list() + for insert_sql in sql_list: + t = threading.Thread(target=tdSql.execute, args=(insert_sql,)) + tlist.append(t) + return tlist + + def multiThreadRun(self, tlist): + for t in tlist: + t.start() + for t in tlist: + t.join() + + def run(self): + tdSql.prepare() + tdSql.execute('create database if not exists test;') + tdSql.execute('create table test.stb (ts timestamp, c11 int, c12 float ) TAGS(t11 int, t12 int );') + tdSql.execute('create table test.tb using test.stb TAGS (1, 1);') + sql_list = list() + for i in range(5): + sql = f'insert into test.tb values (now-{i}m, {i}, {i});' + sql_list.append(sql) + sql_list.append(f'drop database test;') + tlist = self.genMultiThreadSeq(sql_list) + self.multiThreadRun(tlist) + tdSql.query(f'show databases') + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 03e9b15237994f7815b472c2371966b70f557db5 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 18 Jul 2022 13:32:17 +0800 Subject: [PATCH 02/26] fix: fix taosc memory leak --- include/libs/qcom/query.h | 5 +++++ source/client/src/clientHb.c | 4 +--- source/client/src/clientImpl.c | 8 -------- source/client/src/clientMsgHandler.c | 6 ++++++ source/client/src/tmq.c | 2 ++ source/libs/catalog/inc/catalogInt.h | 8 ++++++++ source/libs/catalog/src/ctgRemote.c | 8 ++++---- source/libs/executor/src/dataInserter.c | 1 + source/libs/executor/src/executorimpl.c | 7 +------ source/libs/nodes/src/nodesUtilFuncs.c | 10 +++++++++- source/libs/qcom/src/queryUtil.c | 10 ++++++++++ source/libs/scheduler/src/schRemote.c | 10 ++++++---- source/libs/scheduler/src/schUtil.c | 16 +++++++++------- 13 files changed, 62 insertions(+), 33 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index a93cf1f9b8..58739b4af7 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -162,9 +162,12 @@ typedef struct SRequestConnInfo { SEpSet mgmtEps; } SRequestConnInfo; +typedef void (*__freeFunc)(void *param); + typedef struct SMsgSendInfo { __async_send_cb_fn_t fp; // async callback function STargetInfo target; // for update epset + __freeFunc paramFreeFp; void* param; uint64_t requestId; uint64_t requestObjRefId; @@ -188,6 +191,8 @@ int32_t cleanupTaskQueue(); */ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); +void destroySendMsgInfo(SMsgSendInfo* pMsgBody); + int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, bool persistHandle, void* ctx); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index b52df105b1..6a583842d1 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -286,13 +286,10 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { if (pInst == NULL || NULL == *pInst) { taosThreadMutexUnlock(&appInfo.mutex); tscError("cluster not exist, key:%s", key); - taosMemoryFreeClear(param); tFreeClientHbBatchRsp(&pRsp); return -1; } - taosMemoryFreeClear(param); - if (code != 0) { (*pInst)->onlineDnodes = ((*pInst)->totalDnodes ? 0 : -1); tscDebug("hb rsp error %s, update server status %d/%d", tstrerror(code), (*pInst)->onlineDnodes, (*pInst)->totalDnodes); @@ -716,6 +713,7 @@ static void *hbThreadFunc(void *param) { pInfo->msgInfo.len = tlen; pInfo->msgType = TDMT_MND_HEARTBEAT; pInfo->param = strdup(pAppHbMgr->key); + pInfo->paramFreeFp = taosMemoryFree; pInfo->requestId = generateRequestId(); pInfo->requestObjRefId = 0; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index e8eef0d287..9ad3be4d92 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -29,7 +29,6 @@ static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest); -static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); static bool stringLengthCheck(const char* str, size_t maxsize) { if (str == NULL) { @@ -1215,13 +1214,6 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { return pMsgSendInfo; } -static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { - assert(pMsgBody != NULL); - taosMemoryFreeClear(pMsgBody->target.dbFName); - taosMemoryFreeClear(pMsgBody->msgInfo.pData); - taosMemoryFreeClear(pMsgBody); -} - void updateTargetEpSet(SMsgSendInfo* pSendInfo, STscObj* pTscObj, SRpcMsg* pMsg, SEpSet* pEpSet) { if (NULL == pEpSet) { return; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 520a566e2b..db33532aef 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -255,6 +255,8 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) { catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid); } + taosMemoryFree(pMsg->pData); + if (pRequest->body.queryFp != NULL) { pRequest->body.queryFp(pRequest->body.param, pRequest, code); } else { @@ -278,6 +280,8 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) { pRequest->body.resInfo.execRes.res = alterRsp.pMeta; } + taosMemoryFree(pMsg->pData); + if (pRequest->body.queryFp != NULL) { SExecResult* pRes = &pRequest->body.resInfo.execRes; @@ -387,6 +391,8 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) { tFreeSShowVariablesRsp(&rsp); } + taosMemoryFree(pMsg->pData); + if (pRequest->body.queryFp != NULL) { pRequest->body.queryFp(pRequest->body.param, pRequest, code); } else { diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index ed0ec516b2..fb835d3878 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -506,6 +506,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT pMsgSendInfo->requestId = generateRequestId(); pMsgSendInfo->requestObjRefId = 0; pMsgSendInfo->param = pParam; + pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->fp = tmqCommitCb2; pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET; // send msg @@ -1516,6 +1517,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; sendInfo->param = pParam; + sendInfo->paramFreeFp = taosMemoryFree; sendInfo->fp = tmqAskEpCb; sendInfo->msgType = TDMT_MND_MQ_ASK_EP; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index fb9f588bae..9003de97d7 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -532,6 +532,14 @@ typedef struct SCtgOperation { } \ } while (0) +#define CTG_API_JENTER() do { \ + CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); \ + CTG_LOCK(CTG_READ, &gCtgMgmt.lock); \ + if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { \ + CTG_ERR_JRET(TSDB_CODE_CTG_OUT_OF_SERVICE); \ + } \ +} while (0) + #define CTG_API_LEAVE_NOLOCK(c) do { \ int32_t __code = c; \ diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 8e0a5b7de3..1e375471f9 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -244,10 +244,11 @@ int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t ctgHandleMsgCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { SCtgTaskCallbackParam* cbParam = (SCtgTaskCallbackParam*)param; int32_t code = 0; + SCtgJob* pJob = NULL; - CTG_API_ENTER(); + CTG_API_JENTER(); - SCtgJob* pJob = taosAcquireRef(gCtgMgmt.jobPool, cbParam->refId); + pJob = taosAcquireRef(gCtgMgmt.jobPool, cbParam->refId); if (NULL == pJob) { qDebug("ctg job refId 0x%" PRIx64 " already dropped", cbParam->refId); goto _return; @@ -266,8 +267,6 @@ _return: if (pJob) { taosReleaseRef(gCtgMgmt.jobPool, cbParam->refId); } - - taosMemoryFree(param); CTG_API_LEAVE(code); } @@ -293,6 +292,7 @@ int32_t ctgMakeMsgSendInfo(SCtgTask* pTask, int32_t msgType, SMsgSendInfo **pMsg param->taskId = pTask->taskId; msgSendInfo->param = param; + msgSendInfo->paramFreeFp = taosMemoryFree; msgSendInfo->fp = ctgHandleMsgCallback; *pMsgSendInfo = msgSendInfo; diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index a575e355f1..440a8e014a 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -110,6 +110,7 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs pParam->pInserter = pInserter; pMsgSendInfo->param = pParam; + pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->msgInfo.pData = pMsg; pMsgSendInfo->msgInfo.len = ntohl(pMsg->length); pMsgSendInfo->msgType = TDMT_VND_SUBMIT; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c3d9f66479..5a1fb770e5 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1990,12 +1990,6 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { return TSDB_CODE_SUCCESS; } -static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { - assert(pMsgBody != NULL); - taosMemoryFreeClear(pMsgBody->msgInfo.pData); - taosMemoryFreeClear(pMsgBody); -} - void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; assert(pMsg->info.ahandle != NULL); @@ -2055,6 +2049,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf pWrapper->sourceIndex = sourceIndex; pMsgSendInfo->param = pWrapper; + pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->msgInfo.pData = pMsg; pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); pMsgSendInfo->msgType = pSource->fetchMsgType; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 1dc3db033b..82b6e7b236 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1923,15 +1923,18 @@ int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** return partitionLogicCond(pCondition, pPrimaryKeyCond, pTagIndexCond, pTagCond, pOtherCond); } + bool needOutput = false; switch (classifyCondition(*pCondition)) { case COND_TYPE_PRIMARY_KEY: if (NULL != pPrimaryKeyCond) { *pPrimaryKeyCond = *pCondition; + needOutput = true; } break; case COND_TYPE_TAG_INDEX: if (NULL != pTagIndexCond) { *pTagIndexCond = *pCondition; + needOutput = true; } if (NULL != pTagCond) { SNode* pTempCond = *pCondition; @@ -1942,21 +1945,26 @@ int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** } } *pTagCond = pTempCond; + needOutput = true; } break; case COND_TYPE_TAG: if (NULL != pTagCond) { *pTagCond = *pCondition; + needOutput = true; } break; case COND_TYPE_NORMAL: default: if (NULL != pOtherCond) { *pOtherCond = *pCondition; + needOutput = true; } break; } - *pCondition = NULL; + if (needOutput) { + *pCondition = NULL; + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index eeb44c4f82..6b1476fe46 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -138,6 +138,16 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) return 0; } +void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { + assert(pMsgBody != NULL); + taosMemoryFreeClear(pMsgBody->target.dbFName); + taosMemoryFreeClear(pMsgBody->msgInfo.pData); + if (pMsgBody->paramFreeFp) { + (*pMsgBody->paramFreeFp)(pMsgBody->param); + } + taosMemoryFreeClear(pMsgBody); +} + int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, bool persistHandle, void* rpcCtx) { char* pMsg = rpcMallocCont(pInfo->msgInfo.len); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 6983bbf013..c8eafd7382 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -386,7 +386,6 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { schProcessOnCbEnd(pJob, pTask, code); taosMemoryFreeClear(pMsg->pData); - taosMemoryFreeClear(param); qDebug("end to handle rsp msg, type:%s, handle:%p, code:%s", TMSG_INFO(pMsg->msgType), pMsg->handle, tstrerror(rspCode)); @@ -398,7 +397,6 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); - taosMemoryFreeClear(param); return TSDB_CODE_SUCCESS; } @@ -447,8 +445,8 @@ int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code) { SCH_ERR_JRET(schProcessOnTaskStatusRsp(&rsp.epId, rsp.taskStatus)); _return: + tFreeSSchedulerHbRsp(&rsp); - taosMemoryFree(param); taosMemoryFree(pMsg->pData); SCH_RET(code); } @@ -514,7 +512,9 @@ int32_t schGenerateCallBackInfo(SSchJob *pJob, SSchTask *pTask, void *msg, uint3 SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + msgSendInfo->paramFreeFp = taosMemoryFree; SCH_ERR_JRET(schMakeCallbackParam(pJob, pTask, msgType, isHb, trans, &msgSendInfo->param)); + SCH_ERR_JRET(schGetCallbackFp(msgType, &msgSendInfo->fp)); if (pJob) { @@ -535,7 +535,7 @@ int32_t schGenerateCallBackInfo(SSchJob *pJob, SSchTask *pTask, void *msg, uint3 _return: - schFreeSMsgSendInfo(msgSendInfo); + destroySendMsgInfo(msgSendInfo); SCH_RET(code); } @@ -676,6 +676,7 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { param->pTrans = pJob->conn.pTrans; pMsgSendInfo->param = param; + pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->fp = fp; SRpcCtxVal ctxVal = {.val = pMsgSendInfo, .clone = schCloneSMsgSendInfo}; @@ -795,6 +796,7 @@ int32_t schCloneSMsgSendInfo(void *src, void **dst) { pDst->param = NULL; SCH_ERR_JRET(schCloneCallbackParam(pSrc->param, (SSchCallbackParamHeader **)&pDst->param)); + pDst->paramFreeFp = taosMemoryFree; *dst = pDst; diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index f848dfa210..6f0aca9a8d 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -50,6 +50,12 @@ char* schGetOpStr(SCH_OP_TYPE type) { } } +void schFreeHbTrans(SSchHbTrans *pTrans) { + rpcReleaseHandle(pTrans->trans.pHandle, TAOS_CONN_CLIENT); + + schFreeRpcCtx(&pTrans->rpcCtx); +} + void schCleanClusterHb(void* pTrans) { SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); @@ -57,7 +63,7 @@ void schCleanClusterHb(void* pTrans) { while (hb) { if (hb->trans.pTrans == pTrans) { SQueryNodeEpId* pEpId = taosHashGetKey(hb, NULL); - rpcReleaseHandle(hb->trans.pHandle, TAOS_CONN_CLIENT); + schFreeHbTrans(hb); taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); } @@ -68,8 +74,6 @@ void schCleanClusterHb(void* pTrans) { } int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId) { - return TSDB_CODE_SUCCESS; // TODO ENABLE IT WHEN RPC IS READY - int32_t code = 0; SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); @@ -82,7 +86,7 @@ int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *ep int64_t taskNum = atomic_load_64(&hb->taskNum); if (taskNum <= 0) { - rpcReleaseHandle(hb->trans.pHandle, TAOS_CONN_CLIENT); + schFreeHbTrans(hb); taosHashRemove(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); } SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); @@ -265,9 +269,7 @@ void schFreeRpcCtxVal(const void *arg) { } SMsgSendInfo *pMsgSendInfo = (SMsgSendInfo *)arg; - taosMemoryFreeClear(pMsgSendInfo->param); - taosMemoryFreeClear(pMsgSendInfo->msgInfo.pData); - taosMemoryFreeClear(pMsgSendInfo); + destroySendMsgInfo(pMsgSendInfo); } void schFreeRpcCtx(SRpcCtx *pCtx) { From 34c87dc428f9d1d3551cab8fb243cb0c30f0c2d5 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 18 Jul 2022 15:33:42 +0800 Subject: [PATCH 03/26] fix: ts already exist when insert with schemaless & modify the interface of tmq meta --- include/client/taos.h | 12 ++++++--- source/client/src/clientSml.c | 25 ++++++++++++----- source/client/src/tmq.c | 43 +++++++++++++----------------- source/libs/parser/src/parInsert.c | 4 ++- 4 files changed, 47 insertions(+), 37 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 690c473986..8afefe4e6b 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -259,13 +259,17 @@ enum tmq_res_t { TMQ_RES_TABLE_META = 2, }; +typedef struct { + void* raw_meta; + uint32_t raw_meta_len; + uint16_t raw_meta_type; +} tmq_raw_data; + typedef enum tmq_res_t tmq_res_t; -typedef struct tmq_raw_data tmq_raw_data; DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res); -DLL_EXPORT tmq_raw_data *tmq_get_raw_meta(TAOS_RES *res); -DLL_EXPORT int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data *raw_meta); -DLL_EXPORT void tmq_free_raw_meta(tmq_raw_data *rawMeta); +DLL_EXPORT int32_t tmq_get_raw_meta(TAOS_RES *res, tmq_raw_data *raw_meta); +DLL_EXPORT int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data raw_meta); DLL_EXPORT char *tmq_get_json_meta(TAOS_RES *res); // Returning null means error. Returned result need to be freed by tmq_free_json_meta DLL_EXPORT void tmq_free_json_meta(char* jsonMeta); DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 83f2cf9307..ab3cacfb49 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -268,7 +268,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm *actionNeeded = true; } if (*actionNeeded) { - uDebug("SML:0x%" PRIx64 " generate schema action. column name: %s, action: %d", info->id, colField->name, + uDebug("SML:0x%" PRIx64 " generate schema action. kv->name: %s, action: %d", info->id, kv->key, action->action); } return 0; @@ -436,6 +436,7 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH SSchemaAction *action, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; for (int j = 0; j < taosArrayGetSize(cols); ++j) { + if(j == 0 && !isTag) continue; SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); bool actionNeeded = false; code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, &actionNeeded, info); @@ -452,18 +453,25 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH return TSDB_CODE_SUCCESS; } -static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols) { +static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - for (uint16_t i = 0; i < length; i++) { + int32_t i = 0; + for ( ;i < length; i++) { taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); } - for (int32_t i = 0; i < taosArrayGetSize(cols); i++) { + if (isTag){ + i = 0; + } else { + i = 1; + } + for (; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { return -1; } } + taosHashCleanup(hashTmp); return 0; } @@ -523,7 +531,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { } taosHashClear(hashTmp); - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { + for (uint16_t i = 1; i < pTableMeta->tableInfo.numOfColumns; i++) { taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); } code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &schemaAction, false); @@ -551,12 +559,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { if (needCheckMeta) { code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, - sTableData->tags); + sTableData->tags, true); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, (char *)superTable); goto end; } - code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols); + code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, (char *)superTable); goto end; @@ -832,6 +840,7 @@ static int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArray *cols) { int64_t ts = 0; if (info->protocol == TSDB_SML_LINE_PROTOCOL) { +// uError("SML:data:%s,len:%d", data, len); ts = smlParseInfluxTime(info, data, len); } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { ts = smlParseOpenTsdbTime(info, data, len); @@ -2031,6 +2040,8 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo * static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { SSmlLineInfo elements = {0}; + uError("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s, hello", info->id, sql); + int ret = smlParseInfluxString(sql, &elements, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 6ad6413159..46b21086ce 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -106,12 +106,6 @@ struct tmq_t { tsem_t rspSem; }; -struct tmq_raw_data { - void* raw_meta; - int32_t raw_meta_len; - int16_t raw_meta_type; -}; - enum { TMQ_VG_STATUS__IDLE = 0, TMQ_VG_STATUS__WAIT, @@ -1858,16 +1852,15 @@ const char* tmq_get_table_name(TAOS_RES* res) { return NULL; } -tmq_raw_data *tmq_get_raw_meta(TAOS_RES* res) { - if (TD_RES_TMQ_META(res)) { - tmq_raw_data *raw = taosMemoryCalloc(1, sizeof(tmq_raw_data)); +int32_t tmq_get_raw_meta(TAOS_RES* res, tmq_raw_data *raw) { + if (TD_RES_TMQ_META(res) && raw) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; raw->raw_meta = pMetaRspObj->metaRsp.metaRsp; raw->raw_meta_len = pMetaRspObj->metaRsp.metaRspLen; raw->raw_meta_type = pMetaRspObj->metaRsp.resMsgType; - return raw; + return TSDB_CODE_SUCCESS; } - return NULL; + return TSDB_CODE_INVALID_PARA; } static char *buildCreateTableJson(SSchemaWrapper *schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t){ @@ -2875,23 +2868,23 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ return code; } -int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data *raw_meta){ - if (!taos || !raw_meta) { +int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data raw_meta){ + if (!taos) { return TSDB_CODE_INVALID_PARA; } - if(raw_meta->raw_meta_type == TDMT_VND_CREATE_STB) { - return taosCreateStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_ALTER_STB){ - return taosCreateStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_DROP_STB){ - return taosDropStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_CREATE_TABLE){ - return taosCreateTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_ALTER_TABLE){ - return taosAlterTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_DROP_TABLE){ - return taosDropTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); + if(raw_meta.raw_meta_type == TDMT_VND_CREATE_STB) { + return taosCreateStb(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); + }else if(raw_meta.raw_meta_type == TDMT_VND_ALTER_STB){ + return taosCreateStb(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); + }else if(raw_meta.raw_meta_type == TDMT_VND_DROP_STB){ + return taosDropStb(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); + }else if(raw_meta.raw_meta_type == TDMT_VND_CREATE_TABLE){ + return taosCreateTable(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); + }else if(raw_meta.raw_meta_type == TDMT_VND_ALTER_TABLE){ + return taosAlterTable(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); + }else if(raw_meta.raw_meta_type == TDMT_VND_DROP_TABLE){ + return taosDropTable(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); } return TSDB_CODE_INVALID_PARA; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 5ac4476d14..623b26204d 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -2177,7 +2177,7 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS SSmlKv* kv = taosArrayGetP(cols, i); SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key}; col_id_t t = lastColIdx + 1; - col_id_t index = findCol(&sToken, t, nCols, pSchema); + col_id_t index = (t == 0 ? 0 : findCol(&sToken, t, nCols, pSchema)); if (index < 0 && t > 0) { index = findCol(&sToken, 0, t, pSchema); isOrdered = false; @@ -2401,7 +2401,9 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols } else { int32_t colLen = kv->length; if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { +// uError("SML:data before:%ld, precision:%d", kv->i, pTableMeta->tableInfo.precision); kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); +// uError("SML:data after:%ld, precision:%d", kv->i, pTableMeta->tableInfo.precision); } if (IS_VAR_DATA_TYPE(kv->type)) { From be35f08d094532f89790d48b6032a60206ae91ac Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Mon, 18 Jul 2022 15:52:38 +0800 Subject: [PATCH 04/26] update case about bug fix --- tests/system-test/2-query/abs.py | 3 +++ tests/system-test/2-query/last_row.py | 4 +++- tests/system-test/2-query/max_partition.py | 2 ++ tests/system-test/2-query/sample.py | 5 ++++- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index 6dc65ce3c2..f0f559c6aa 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -554,6 +554,9 @@ class TDTestCase: tdSql.query("select t1 from stb1 where abs(c1+t1)=1") tdSql.checkRows(1) tdSql.checkData(0,0,0) + + tdSql.query("select last_row(c1) from (select ts , c1 ,t1 from stb1)") + tdSql.checkRows(20) tdSql.query( "select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1") diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index fce3cc7043..6a75d1a1c0 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -604,7 +604,7 @@ class TDTestCase: tdSql.checkData(0,0,None) tdSql.query("select ts , last_row(c1) ,c1 from (select ts , c1 ,t1 from stb1)") - tdSql.checkData(0,1,None,None) + tdSql.checkData(0,1,None) tdSql.query("select ts , last_row(c1) ,c1 from (select ts , max(c1) c1 ,t1 from stb1 where ts >now -1h and ts ="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s)') tdSql.checkRows(5) + tdSql.query("select unique(c1) from stb1 partition by tbname") + def support_super_table_test(self): diff --git a/tests/system-test/2-query/max_partition.py b/tests/system-test/2-query/max_partition.py index cf0d6639c2..5fa78c08e4 100644 --- a/tests/system-test/2-query/max_partition.py +++ b/tests/system-test/2-query/max_partition.py @@ -162,6 +162,8 @@ class TDTestCase: tdSql.query("select tbname , max(c1) from stb partition by tbname interval(10s)") tdSql.checkRows(self.row_nums*2) + tdSql.query("select unique(c1) from stb1 partition by tbname order by tbname") + tdSql.query("select tbname , count(c1) from sub_stb_1 partition by tbname interval(10s)") tdSql.checkData(0,0,'sub_stb_1') tdSql.checkData(0,1,self.row_nums) diff --git a/tests/system-test/2-query/sample.py b/tests/system-test/2-query/sample.py index 84ff5a0056..b6bdd8926e 100644 --- a/tests/system-test/2-query/sample.py +++ b/tests/system-test/2-query/sample.py @@ -870,7 +870,10 @@ class TDTestCase: tdSql.query("select sample(c1 ,1000) from st") tdSql.checkRows(1000) - + # bug need fix + tdSql.query("select c1 ,t1, sample(c1,2) from db.stb1 partition by c1 ") + tdSql.query("select sample(c1,2) from db.stb1 partition by c1 ") + # tdSql.query("select c1 ,ind, sample(c1,2) from sample_db.st partition by c1 ") def run(self): import traceback From 1d30ff6190b710ad4b01ab74d5ea5d49260967ba Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Mon, 18 Jul 2022 16:36:13 +0800 Subject: [PATCH 05/26] update case for bug fix --- tests/system-test/2-query/csum.py | 4 +-- tests/system-test/2-query/max_partition.py | 35 +++++++++++++++++++++- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/csum.py b/tests/system-test/2-query/csum.py index 425597a919..91869cb012 100644 --- a/tests/system-test/2-query/csum.py +++ b/tests/system-test/2-query/csum.py @@ -435,8 +435,8 @@ class TDTestCase: tdSql.checkRows(40) # # bug need fix - # tdSql.query("select csum(st1+c1) from stb1 partition by tbname slimit 1 ") - # tdSql.checkRows(4) + tdSql.query("select csum(st1+c1) from stb1 partition by tbname slimit 1 ") + tdSql.checkRows(4) # tdSql.error("select csum(st1+c1) from stb1 partition by tbname limit 1 ") diff --git a/tests/system-test/2-query/max_partition.py b/tests/system-test/2-query/max_partition.py index 5fa78c08e4..0a7214ec75 100644 --- a/tests/system-test/2-query/max_partition.py +++ b/tests/system-test/2-query/max_partition.py @@ -162,12 +162,45 @@ class TDTestCase: tdSql.query("select tbname , max(c1) from stb partition by tbname interval(10s)") tdSql.checkRows(self.row_nums*2) - tdSql.query("select unique(c1) from stb1 partition by tbname order by tbname") + tdSql.query("select unique(c1) from stb partition by tbname order by tbname") tdSql.query("select tbname , count(c1) from sub_stb_1 partition by tbname interval(10s)") tdSql.checkData(0,0,'sub_stb_1') tdSql.checkData(0,1,self.row_nums) + tdSql.query("select c1 , mavg(c1 ,2 ) from stb partition by c1") + tdSql.checkRows(72) + + tdSql.query("select c1 , diff(c1 , 0) from stb partition by c1") + tdSql.checkRows(72) + + tdSql.query("select c1 , csum(c1) from stb partition by c1") + tdSql.checkRows(80) + + tdSql.query("select c1 , sample(c1,2) from stb partition by c1 order by c1") + tdSql.checkRows(21) + # bug need fix + # tdSql.checkData(0,1,None) + + tdSql.query("select c1 , twa(c1) from stb partition by c1 order by c1") + tdSql.checkRows(11) + tdSql.checkData(0,1,0.000000000) + + tdSql.query("select c1 , irate(c1) from stb partition by c1 order by c1") + tdSql.checkRows(11) + tdSql.checkData(0,1,None) + + tdSql.query("select c1 , DERIVATIVE(c1,2,1) from stb partition by c1 order by c1") + tdSql.checkRows(72) + # bug need fix + # tdSql.checkData(0,1,None) + + + + + + + # bug need fix # tdSql.query(" select tbname , max(c1) from stb partition by tbname order by tbname slimit 5 soffset 0 ") # tdSql.checkRows(5) From cbfc1688f40794e1c8931f257acdeb229c2726cb Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Mon, 18 Jul 2022 16:38:57 +0800 Subject: [PATCH 06/26] update case --- tests/system-test/2-query/abs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index f0f559c6aa..f924cb7c3d 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -555,8 +555,8 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0,0,0) - tdSql.query("select last_row(c1) from (select ts , c1 ,t1 from stb1)") - tdSql.checkRows(20) + tdSql.query("select abs(c1) from (select ts , c1 ,t1 from stb1)") + tdSql.checkRows(25) tdSql.query( "select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1") From ccdda33ff8012cd593f733846b669f0c389757e7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 18 Jul 2022 17:17:51 +0800 Subject: [PATCH 07/26] fix:modify the interface of tmq meta --- examples/c/tmq.c | 6 +++--- tests/test/c/tmqSim.c | 46 +++++++++++++++++++++---------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 38c3eba647..7e9a24ff1f 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -28,8 +28,9 @@ static void msg_process(TAOS_RES* msg) { printf("db: %s\n", tmq_get_db_name(msg)); printf("vg: %d\n", tmq_get_vgroup_id(msg)); if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { - tmq_raw_data* raw = tmq_get_raw_meta(msg); - if (raw) { + tmq_raw_data raw = {0}; + int32_t code = tmq_get_raw_meta(msg, &raw); + if (code == 0) { TAOS* pConn = taos_connect("192.168.1.86", "root", "taosdata", NULL, 0); if (pConn == NULL) { return; @@ -53,7 +54,6 @@ static void msg_process(TAOS_RES* msg) { printf("write raw data: %s\n", tmq_err2str(ret)); taos_close(pConn); } - tmq_free_raw_meta(raw); char* result = tmq_get_json_meta(msg); if (result) { printf("meta result: %s\n", result); diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 5459e3f159..70b8c1257e 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -45,10 +45,10 @@ typedef enum { NOTIFY_CMD_ID_BUTT, } NOTIFY_CMD_ID; -typedef enum enumQUERY_TYPE { - NO_INSERT_TYPE, - INSERT_TYPE, - QUERY_TYPE_BUT +typedef enum enumQUERY_TYPE { + NO_INSERT_TYPE, + INSERT_TYPE, + QUERY_TYPE_BUT } QUERY_TYPE; typedef struct { @@ -597,9 +597,10 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn tmq_get_topic_name(msg), vgroupId); { - tmq_raw_data *raw = tmq_get_raw_meta(msg); + tmq_raw_data raw = {0}; + int32_t code = tmq_get_raw_meta(msg, &raw); - if(raw){ + if(code == TSDB_CODE_SUCCESS){ TAOS_RES* pRes = taos_query(pInfo->taos, "use metadb"); if (taos_errno(pRes) != 0) { pError("error when use metadb, reason:%s\n", taos_errstr(pRes)); @@ -609,10 +610,9 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn exit(-1); } taos_free_result(pRes); - taosFprintfFile(g_fp, "raw:%p\n", raw); + taosFprintfFile(g_fp, "raw:%p\n", &raw); - int32_t ret = taos_write_raw_meta(pInfo->taos, raw); - taosMemoryFree(raw); + taos_write_raw_meta(pInfo->taos, raw); } char* result = tmq_get_json_meta(msg); @@ -1169,23 +1169,23 @@ void* ombConsumeThreadFunc(void* param) { static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type) { - TAOS_RES *res = taos_query(taos, command); - int32_t code = taos_errno(res); - - if (code != 0) { + TAOS_RES *res = taos_query(taos, command); + int32_t code = taos_errno(res); + + if (code != 0) { pPrint("%s Failed to execute <%s>, reason: %s %s", GREEN, command, taos_errstr(res), NC); taos_free_result(res); return -1; - } - - if (INSERT_TYPE == type) { - int affectedRows = taos_affected_rows(res); - taos_free_result(res); - return affectedRows; - } - - taos_free_result(res); - return 0; + } + + if (INSERT_TYPE == type) { + int affectedRows = taos_affected_rows(res); + taos_free_result(res); + return affectedRows; + } + + taos_free_result(res); + return 0; } void* ombProduceThreadFunc(void* param) { From 322ec02e39e711cd243f0d2fc27a05fa40d24373 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 18 Jul 2022 11:45:13 +0000 Subject: [PATCH 08/26] fix: delete causes coredump --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbFS.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFile.c | 2 ++ source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 3f667a7465..4e2c762cd3 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -186,6 +186,7 @@ int32_t tsdbGetNRowsInTbData(STbData *pTbData); typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT; void tsdbDataFileName(STsdb *pTsdb, SDFileSet *pDFileSet, EDataFileT ftype, char fname[]); bool tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT ftype); +bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2); int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype); int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype); int32_t tPutDataFileHdr(uint8_t *p, SDFileSet *pSet, EDataFileT ftype); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 70fbd90646..3bc79621e1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -245,7 +245,7 @@ static int32_t tsdbApplyDelFileChange(STsdbFS *pFS, SDelFile *pFrom, SDelFile *p char fname[TSDB_FILENAME_LEN]; if (pFrom && pTo) { - if (pFrom != pTo) { + if (!tsdbDelFileIsSame(pFrom, pTo)) { tsdbDelFileName(pFS->pTsdb, pFrom, fname); if (taosRemoveFile(fname) < 0) { code = TAOS_SYSTEM_ERROR(errno); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index e7f8cb4789..f15ad072e7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -140,6 +140,8 @@ bool tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT fty } } +bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2) { return pDelFile1->commitID == pDelFile2->commitID; } + int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype) { int32_t code = 0; int64_t n; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 0ed2e12542..5e8157864f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -246,7 +246,7 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb tsdbDelFileName(pTsdb, pFile, fname); pDelFReader->pReadH = taosOpenFile(fname, TD_FILE_READ); - if (pDelFReader == NULL) { + if (pDelFReader->pReadH == NULL) { code = TAOS_SYSTEM_ERROR(errno); taosMemoryFree(pDelFReader); goto _err; From 4c00aec172663e7d7998071aac72348a2032c484 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 18 Jul 2022 21:26:29 +0800 Subject: [PATCH 09/26] test: add tmq test case --- tests/system-test/7-tmq/tmqCommon.py | 7 +++++++ tests/system-test/7-tmq/tmqUpdate-1ctb.py | 13 +++++++++++-- tests/system-test/fulltest.sh | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 151558a33a..1f57099e3a 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -205,6 +205,13 @@ class TMQCom: tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName)) return + def drop_ctable(self, tsql, dbname=None, count=1, default_ctbname_prefix="ctb",ctbStartIdx=0): + for _ in range(count): + create_ctable_sql = f'drop table {dbname}.{default_ctbname_prefix}{ctbStartIdx};' + ctbStartIdx += 1 + tdLog.info("drop ctb sql: %s"%create_ctable_sql) + tsql.execute(create_ctable_sql) + # schema: (ts timestamp, c1 int, c2 binary(16)) def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None): tdLog.debug("start to insert data ............") diff --git a/tests/system-test/7-tmq/tmqUpdate-1ctb.py b/tests/system-test/7-tmq/tmqUpdate-1ctb.py index de11f0f9ab..5d891bdedf 100644 --- a/tests/system-test/7-tmq/tmqUpdate-1ctb.py +++ b/tests/system-test/7-tmq/tmqUpdate-1ctb.py @@ -116,7 +116,12 @@ class TDTestCase: # paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl consumerId = 0 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 3/2) + + if self.snapshot == 0: + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/2)) + elif self.snapshot == 1: + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1)) + topicList = topicFromStb1 ifcheckdata = 1 ifManualCommit = 1 @@ -199,7 +204,11 @@ class TDTestCase: # paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl consumerId = 1 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) + if self.snapshot == 0: + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (2)) + elif self.snapshot == 1: + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1)) + topicList = topicFromStb1 ifcheckdata = 1 ifManualCommit = 1 diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index d98764edb6..24d1be203d 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -182,7 +182,7 @@ python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py #python3 ./test.py -f 7-tmq/tmqDnodeRestart.py -#python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py +python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb.py #python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py From 85fd5566140b6ebb0c7ea6f4b08fa3771aab4e32 Mon Sep 17 00:00:00 2001 From: tomchon Date: Mon, 18 Jul 2022 22:47:46 +0800 Subject: [PATCH 10/26] test: modify testcases of muti-mnodes --- tests/system-test/2-query/tsbsQuery.py | 12 +- tests/system-test/6-cluster/5dnode1mnode.py | 2 +- .../5dnode3mnodeRestartDnodeInsertData.py | 4 +- .../5dnode3mnodeSep1VnodeStopMnodeCreateDb.py | 16 +- ...ode3mnodeSep1VnodeStopMnodeCreateDbRep3.py | 180 ++++++++++++++++++ ...5dnode3mnodeSep1VnodeStopMnodeCreateStb.py | 3 +- .../5dnode3mnodeSep1VnodeStopVnodeCreateDb.py | 77 ++++---- .../6-cluster/clusterCommonCheck.py | 2 +- .../6-cluster/clusterCommonCreate.py | 2 + 9 files changed, 250 insertions(+), 48 deletions(-) create mode 100644 tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py index 664bc8dc5b..d766bc5089 100644 --- a/tests/system-test/2-query/tsbsQuery.py +++ b/tests/system-test/2-query/tsbsQuery.py @@ -113,6 +113,7 @@ class TDTestCase: #taosc core dumped tdSql.execute("create table random_measure2_1 (ts timestamp,ela float, name binary(40))") tdSql.query("SELECT ts,diff(mv) AS difka FROM (SELECT ts,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name,ts interval(10m) fill(value,0)) GROUP BY name,ts;") + tdSql.query("select name,diff(mv) AS difka FROM (SELECT ts,name,mv FROM (SELECT _wstart as ts,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name interval(10m) fill(value,0))) group BY name ;") tdSql.query("SELECT _wstart,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name interval(10m) fill(value,0)") # 7. avg-load @@ -124,9 +125,16 @@ class TDTestCase: #it's already supported: # last-loc - tdSql.query("") + tdSql.query("SELECT last_row(ts),latitude,longitude,name,driver FROM readings WHERE fleet='South' and name IS NOT NULL partition BY name,driver order by name ;") - def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + + #2. low-fuel + tdSql.query("SELECT last_row(ts),name,driver,fuel_state,driver FROM diagnostics WHERE fuel_state <= 0.1 AND fleet = 'South' and name IS NOT NULL GROUP BY name,driver order by name;") + + # 3. avg-vs-projected-fuel-consumption + tdSql.query("select avg(fuel_consumption) as avg_fuel_consumption,avg(nominal_fuel_consumption) as nominal_fuel_consumption from readings where velocity > 1 group by fleet") + + def run(self): tdLog.printNoPrefix("==========step1:create database and table,insert data ==============") self.prepareData() self.tsbsIotQuery() diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py index 391cf3396d..ee2d8afb81 100644 --- a/tests/system-test/6-cluster/5dnode1mnode.py +++ b/tests/system-test/6-cluster/5dnode1mnode.py @@ -126,7 +126,7 @@ class TDTestCase: tdSql.error("alter database db strict 'off'") # tdSql.execute('alter database db strict 'on'') # tdSql.query('show databases;') - # tdSql.checkData(2,5,'strict') + # tdSql.checkData(2,5,'on') def getConnection(self, dnode): host = dnode.cfgDict["fqdn"] diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py index e5946342d2..587049e44e 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py @@ -107,7 +107,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbNum': 200, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - "rowsPerTbl": 1000, + "rowsPerTbl": 100, "batchNum": 5000 } @@ -213,7 +213,7 @@ class TDTestCase: tdSql.checkRows(rowsPerStb) def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=2,stopRole='dnode') + self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=1,stopRole='dnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py index f820d812ec..2a4e86db49 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py @@ -68,7 +68,7 @@ class TDTestCase: def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db', - 'dbNumbers': 20, + 'dbNumbers': 6, 'dropFlag': 1, 'event': '', 'vgroups': 4, @@ -110,7 +110,7 @@ class TDTestCase: print(tdSql.queryResult) clusterComCheck.checkDnodes(dnodeNumbers) - # create database and stable + tdLog.info("create database and stable") tdDnodes=cluster.dnodes stopcount =0 threads=[] @@ -156,10 +156,16 @@ class TDTestCase: for tr in threads: tr.join() + tdLog.info("check dnode number:") clusterComCheck.checkDnodes(dnodeNumbers) - clusterComCheck.checkDbRows(allDbNumbers) - for i in range(restartNumbers): - clusterComCheck.checkDb(paraDict['dbNumbers'],restartNumbers,dbNameIndex = '%s%d'%(paraDict["dbName"],i)) + tdSql.query("show databases") + tdLog.debug("we find %d databases but exepect to create %d databases "%(tdSql.queryRows-2,allDbNumbers-2)) + + # tdLog.info("check DB Rows:") + # clusterComCheck.checkDbRows(allDbNumbers) + # tdLog.info("check DB Status on by on") + # for i in range(restartNumbers): + # clusterComCheck.checkDb(paraDict['dbNumbers'],restartNumbers,dbNameIndex = '%s%d'%(paraDict["dbName"],i)) def run(self): diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py new file mode 100644 index 0000000000..1fa77d3bfd --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py @@ -0,0 +1,180 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +class TDTestCase: + + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _async_raise(self, tid, exctype): + """raises the exception, performs cleanup if needed""" + if not inspect.isclass(exctype): + exctype = type(exctype) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) + if res == 0: + raise ValueError("invalid thread id") + elif res != 1: + # """if it returns a number greater than one, you're in trouble, + # and you should call it again with exc=NULL to revert the effect""" + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + raise SystemError("PyThreadState_SetAsyncExc failed") + + def stopThread(self,thread): + self._async_raise(thread.ident, SystemExit) + + + def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db', + 'dbNumbers': 6, + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 3, + 'stbName': 'stb', + 'stbNumbers': 100, + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + } + + dnodeNumbers=int(dnodeNumbers) + mnodeNums=int(mnodeNums) + vnodeNumbers = int(dnodeNumbers-mnodeNums) + allDbNumbers=(paraDict['dbNumbers']*restartNumbers) + allStbNumbers=(paraDict['stbNumbers']*restartNumbers) + + tdLog.info("first check dnode and mnode") + tdSql.query("show dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodeNumbers) + clusterComCheck.checkMnodeStatus(1) + + # fisr add three mnodes; + tdLog.info("fisr add three mnodes and check mnode status") + tdSql.execute("create mnode on dnode 2") + clusterComCheck.checkMnodeStatus(2) + tdSql.execute("create mnode on dnode 3") + clusterComCheck.checkMnodeStatus(3) + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("show dnodes;") + print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodeNumbers) + + tdLog.info("create database and stable") + tdDnodes=cluster.dnodes + stopcount =0 + threads=[] + for i in range(restartNumbers): + dbNameIndex = '%s%d'%(paraDict["dbName"],i) + newTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=clusterComCreate.create_databases, args=(newTdSql, dbNameIndex,paraDict["dbNumbers"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) + + for tr in threads: + tr.start() + + tdLog.info("Take turns stopping Mnodes ") + while stopcount < restartNumbers: + tdLog.info(" restart loop: %d"%stopcount ) + if stopRole == "mnode": + for i in range(mnodeNums): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + elif stopRole == "vnode": + for i in range(vnodeNumbers): + tdDnodes[i+mnodeNums].stoptaosd() + # sleep(10) + tdDnodes[i+mnodeNums].starttaosd() + # sleep(10) + elif stopRole == "dnode": + for i in range(dnodeNumbers): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + + # dnodeNumbers don't include database of schema + if clusterComCheck.checkDnodes(dnodeNumbers): + tdLog.info("check dnodes status is ready") + else: + tdLog.info("check dnodes status is not ready") + self.stopThread(threads) + tdLog.exit("one or more of dnodes failed to start ") + # self.check3mnode() + stopcount+=1 + + for tr in threads: + tr.join() + tdLog.info("check dnode number:") + clusterComCheck.checkDnodes(dnodeNumbers) + tdSql.query("show databases") + tdLog.debug("we find %d databases but exepect to create %d databases "%(tdSql.queryRows-2,allDbNumbers-2)) + + # tdLog.info("check DB Rows:") + # clusterComCheck.checkDbRows(allDbNumbers) + # tdLog.info("check DB Status on by on") + # for i in range(restartNumbers): + # clusterComCheck.checkDb(paraDict['dbNumbers'],restartNumbers,dbNameIndex = '%s%d'%(paraDict["dbName"],i)) + + + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=2,stopRole='mnode') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py index 29fb3008c3..f7d62857f9 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py @@ -189,7 +189,8 @@ class TDTestCase: tdSql.execute("use %s" %(paraDict["dbName"])) tdSql.query("show stables") - tdSql.checkRows(allStbNumbers) + tdLog.debug("we find %d stables but exepect to create %d stables "%(tdSql.queryRows,allStbNumbers)) + # tdSql.checkRows(allStbNumbers) def run(self): diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py index 80508979c5..51bd12410f 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py @@ -68,10 +68,10 @@ class TDTestCase: def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db', - 'dbNumbers': 20, + 'dbNumbers': 8, 'dropFlag': 1, 'event': '', - 'vgroups': 4, + 'vgroups': 2, 'replica': 1, 'stbName': 'stb', 'stbNumbers': 100, @@ -124,49 +124,54 @@ class TDTestCase: for tr in threads: tr.start() - tdLog.info("Take turns stopping Mnodes ") - while stopcount < restartNumbers: - tdLog.info(" restart loop: %d"%stopcount ) - if stopRole == "mnode": - for i in range(mnodeNums): - tdDnodes[i].stoptaosd() - # sleep(10) - tdDnodes[i].starttaosd() - # sleep(10) - elif stopRole == "vnode": - for i in range(vnodeNumbers): - tdDnodes[i+mnodeNums].stoptaosd() - # sleep(10) - tdDnodes[i+mnodeNums].starttaosd() - # sleep(10) - elif stopRole == "dnode": - for i in range(dnodeNumbers): - tdDnodes[i].stoptaosd() - # sleep(10) - tdDnodes[i].starttaosd() - # sleep(10) + # tdLog.info("Take turns stopping Mnodes ") + # while stopcount < restartNumbers: + # tdLog.info(" restart loop: %d"%stopcount ) + # if stopRole == "mnode": + # for i in range(mnodeNums): + # tdDnodes[i].stoptaosd() + # # sleep(10) + # tdDnodes[i].starttaosd() + # # sleep(10) + # elif stopRole == "vnode": + # for i in range(vnodeNumbers): + # tdDnodes[i+mnodeNums].stoptaosd() + # # sleep(10) + # tdDnodes[i+mnodeNums].starttaosd() + # # sleep(10) + # elif stopRole == "dnode": + # for i in range(dnodeNumbers): + # tdDnodes[i].stoptaosd() + # # sleep(10) + # tdDnodes[i].starttaosd() + # # sleep(10) - # dnodeNumbers don't include database of schema - if clusterComCheck.checkDnodes(dnodeNumbers): - tdLog.info("check dnodes status is ready") - else: - tdLog.info("check dnodes status is not ready") - self.stopThread(threads) - tdLog.exit("one or more of dnodes failed to start ") - # self.check3mnode() - stopcount+=1 + # # dnodeNumbers don't include database of schema + # if clusterComCheck.checkDnodes(dnodeNumbers): + # tdLog.info("check dnodes status is ready") + # else: + # tdLog.info("check dnodes status is not ready") + # self.stopThread(threads) + # tdLog.exit("one or more of dnodes failed to start ") + # # self.check3mnode() + # stopcount+=1 for tr in threads: tr.join() clusterComCheck.checkDnodes(dnodeNumbers) - clusterComCheck.checkDbRows(allDbNumbers) - for i in range(restartNumbers): - clusterComCheck.checkDb(paraDict['dbNumbers'],restartNumbers,dbNameIndex = '%s%d'%(paraDict["dbName"],i)) + tdSql.query("show databases") + tdLog.debug("we find %d databases but exepect to create %d databases "%(tdSql.queryRows-2,allDbNumbers)) + + # # tdLog.info("check DB Rows:") + # clusterComCheck.checkDbRows(allDbNumbers) + # # tdLog.info("check DB Status on by on") + # for i in range(restartNumbers): + # clusterComCheck.checkDb(paraDict['dbNumbers'],restartNumbers,dbNameIndex = '%s%d'%(paraDict["dbName"],i)) def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=2,stopRole='vnode') + self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=15,stopRole='vnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py index cde09b4c58..203d756353 100644 --- a/tests/system-test/6-cluster/clusterCommonCheck.py +++ b/tests/system-test/6-cluster/clusterCommonCheck.py @@ -89,7 +89,7 @@ class ClusterComCheck: # print(query_status) count+=1 if query_status == dbNumbers: - tdLog.success("we find cluster with %d dnode and check all databases are ready within 5s! " %dbNumbers) + tdLog.success(" check %d database and all databases are ready within 5s! " %dbNumbers) return True else: tdLog.debug(tdSql.queryResult) diff --git a/tests/system-test/6-cluster/clusterCommonCreate.py b/tests/system-test/6-cluster/clusterCommonCreate.py index 851fe3b51c..667e5e383e 100644 --- a/tests/system-test/6-cluster/clusterCommonCreate.py +++ b/tests/system-test/6-cluster/clusterCommonCreate.py @@ -127,6 +127,7 @@ class ClusterComCreate: for i in range(dbNumbers): if dropFlag == 1: tsql.execute("drop database if exists %s_%d"%(dbNameIndex,i)) + tdLog.debug("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,i, vgroups, replica)) tsql.execute("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,i, vgroups, replica)) tdLog.debug("complete to create database %s_%d"%(dbNameIndex,i)) return @@ -138,6 +139,7 @@ class ClusterComCreate: def create_stables(self,tsql,dbNameIndex,stbNameIndex,stbNumbers): for i in range(stbNumbers): + tdLog.debug("create table if not exists %s.%s_%d (ts timestamp, c1 int, c2 int, c3 binary(16)) tags(t1 int, t2 binary(32))"%(dbNameIndex, stbNameIndex,i)) tsql.execute("create table if not exists %s.%s_%d (ts timestamp, c1 int, c2 int, c3 binary(16)) tags(t1 int, t2 binary(32))"%(dbNameIndex, stbNameIndex,i)) tdLog.debug("complete to create %s.%s_%d" %(dbNameIndex, stbNameIndex,i)) return From 59cc090238ea8ad2fee054624baece20478d553f Mon Sep 17 00:00:00 2001 From: tomchon Date: Mon, 18 Jul 2022 22:49:37 +0800 Subject: [PATCH 11/26] test: modify testcases of muti-mnodes --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index ddbbefe739..34b7767503 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -134,7 +134,7 @@ python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 +# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 5 -M 3 # python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 5 -M 3 From 8d5bd9a83aa5446e409c548248a1b53690260c15 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 19 Jul 2022 09:05:17 +0800 Subject: [PATCH 12/26] fix: fix crash issue --- source/client/src/tmq.c | 1 - source/libs/executor/src/executorimpl.c | 1 - source/libs/scheduler/inc/schInt.h | 3 +-- source/libs/scheduler/src/schJob.c | 3 ++- source/libs/scheduler/src/schRemote.c | 3 +-- source/libs/scheduler/src/schUtil.c | 9 --------- 6 files changed, 4 insertions(+), 16 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index fb835d3878..71c53c9091 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -1517,7 +1517,6 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; sendInfo->param = pParam; - sendInfo->paramFreeFp = taosMemoryFree; sendInfo->fp = tmqAskEpCb; sendInfo->msgType = TDMT_MND_MQ_ASK_EP; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5a1fb770e5..746dc79510 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1986,7 +1986,6 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { tsem_post(&pExchangeInfo->ready); taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId); - taosMemoryFree(pWrapper); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 65b45cc612..02e878f4f8 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -277,7 +277,7 @@ typedef struct SSchJob { bool fetched; int32_t resNumOfRows; SSchResInfo userRes; - const char *sql; + char *sql; SQueryProfileSummary summary; } SSchJob; @@ -461,7 +461,6 @@ int32_t schJobFetchRows(SSchJob *pJob); int32_t schJobFetchRowsA(SSchJob *pJob); int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId); int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId* pEpId, SArray* pStatusList); -void schFreeSMsgSendInfo(SMsgSendInfo *msgSendInfo); char* schGetOpStr(SCH_OP_TYPE type); int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 13a369fac9..de19e7907b 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -675,6 +675,7 @@ void schFreeJobImpl(void *job) { taosMemoryFreeClear(pJob->userRes.execRes); taosMemoryFreeClear(pJob->fetchRes); + taosMemoryFreeClear(pJob->sql); taosMemoryFree(pJob); int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1); @@ -718,7 +719,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->attr.explainMode = pReq->pDag->explainInfo.mode; pJob->conn = *pReq->pConn; - pJob->sql = pReq->sql; + pJob->sql = strdup(pReq->sql); pJob->pDag = pReq->pDag; pJob->chkKillFp = pReq->chkKillFp; pJob->chkKillParam = pReq->chkKillParam; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index c8eafd7382..5452ca31a5 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -863,8 +863,7 @@ _return: } if (pMsgSendInfo) { - taosMemoryFreeClear(pMsgSendInfo->param); - taosMemoryFreeClear(pMsgSendInfo); + destroySendMsgInfo(pMsgSendInfo); } SCH_RET(code); diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 6f0aca9a8d..6f12780ff9 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -292,15 +292,6 @@ void schFreeRpcCtx(SRpcCtx *pCtx) { } } -void schFreeSMsgSendInfo(SMsgSendInfo *msgSendInfo) { - if (NULL == msgSendInfo) { - return; - } - - taosMemoryFree(msgSendInfo->param); - taosMemoryFree(msgSendInfo); -} - int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) { int32_t s = taosHashGetSize(pTaskList); if (s <= 0) { From 7834655c93f9b06b12aacdbf960ed0a9a9eb373b Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Tue, 19 Jul 2022 09:14:13 +0800 Subject: [PATCH 13/26] update test case --- tests/system-test/0-others/sysinfo.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/system-test/0-others/sysinfo.py b/tests/system-test/0-others/sysinfo.py index f0db151165..129c8bc530 100644 --- a/tests/system-test/0-others/sysinfo.py +++ b/tests/system-test/0-others/sysinfo.py @@ -24,7 +24,7 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.dbname = 'db' - self.delaytime = 10 + self.delaytime = 3 def get_database_info(self): tdSql.query('select database()') tdSql.checkData(0,0,None) @@ -42,12 +42,13 @@ class TDTestCase: tdSql.checkData(0,0,version_info) def get_server_status(self): + sleep(self.delaytime) tdSql.query('select server_status()') tdSql.checkData(0,0,1) #!for bug - # tdDnodes.stoptaosd(1) - # sleep(self.delaytime) - # tdSql.error('select server_status()') + tdDnodes.stoptaosd(1) + sleep(self.delaytime) + tdSql.error('select server_status()') def run(self): self.get_database_info() From d3be1e5f4d173311b35a044d7d9355a184d4ddf7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 19 Jul 2022 09:39:26 +0800 Subject: [PATCH 14/26] fix: avoid rpc mem leak --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 62f1f9e315..27a4056249 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -49,9 +49,9 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { dmUpdateEps(pMgmt->pData, statusRsp.pDnodeEps); } } - rpcFreeCont(pRsp->pCont); tFreeSStatusRsp(&statusRsp); } + rpcFreeCont(pRsp->pCont); } void dmSendStatusReq(SDnodeMgmt *pMgmt) { From 802c86fe178a6aff932059560bacc91dbbf3d36b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 19 Jul 2022 09:58:51 +0800 Subject: [PATCH 15/26] fix:error when bind ts column in schemaless --- source/client/src/clientSml.c | 2 +- source/libs/parser/src/parInsert.c | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index ab3cacfb49..49f26e2c24 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2040,7 +2040,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo * static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { SSmlLineInfo elements = {0}; - uError("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s, hello", info->id, sql); + uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s, hello", info->id, sql); int ret = smlParseInfluxString(sql, &elements, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 623b26204d..a6862598ee 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -2161,7 +2161,7 @@ static void smlDestroyTableHandle(void* pHandle) { tdDestroySVCreateTbReq(&handle->createTblReq); } -static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema) { +static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema, bool isTag) { col_id_t nCols = pColList->numOfCols; pColList->numOfBound = 0; @@ -2177,7 +2177,8 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS SSmlKv* kv = taosArrayGetP(cols, i); SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key}; col_id_t t = lastColIdx + 1; - col_id_t index = (t == 0 ? 0 : findCol(&sToken, t, nCols, pSchema)); + col_id_t index = ((t == 0 && !isTag) ? 0 : findCol(&sToken, t, nCols, pSchema)); + uDebug("SML, index:%d, t:%d, ncols:%d, kv->name:%s", index, t, nCols, kv->key); if (index < 0 && t > 0) { index = findCol(&sToken, 0, t, pSchema); isOrdered = false; @@ -2312,7 +2313,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols smlDestroyTableHandle(&smlHandle->tableExecHandle); // free for each table SSchema* pTagsSchema = getTableTagSchema(pTableMeta); setBoundColumnInfo(&smlHandle->tableExecHandle.tags, pTagsSchema, getNumOfTags(pTableMeta)); - int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema); + int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, true); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "bound tags error"); return ret; @@ -2343,7 +2344,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols SSchema* pSchema = getTableColumnSchema(pTableMeta); - ret = smlBoundColumnData(colsSchema, &pDataBlock->boundColumnInfo, pSchema); + ret = smlBoundColumnData(colsSchema, &pDataBlock->boundColumnInfo, pSchema, false); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "bound cols error"); return ret; From 666c6475ee061fe8667dbaab4a0ba8a4ceca8722 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 19 Jul 2022 10:27:09 +0800 Subject: [PATCH 16/26] fix: fix sql crash issue --- source/libs/executor/src/dataInserter.c | 1 - source/libs/scheduler/src/schJob.c | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 440a8e014a..1c08fafaa3 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -91,7 +91,6 @@ _return: tsem_post(&pInserter->ready); taosMemoryFree(pMsg->pData); - taosMemoryFree(param); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index de19e7907b..50f8b91023 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -719,7 +719,9 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->attr.explainMode = pReq->pDag->explainInfo.mode; pJob->conn = *pReq->pConn; - pJob->sql = strdup(pReq->sql); + if (pReq->sql) { + pJob->sql = strdup(pReq->sql); + } pJob->pDag = pReq->pDag; pJob->chkKillFp = pReq->chkKillFp; pJob->chkKillParam = pReq->chkKillParam; From 651ae5832e8a68f2333d060b41c3e8ef7b523871 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 19 Jul 2022 10:46:47 +0800 Subject: [PATCH 17/26] refactor(sync): delete not exist log --- source/libs/sync/src/syncRaftLog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index edc01c9a05..7f905f7cb5 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -313,7 +313,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, snprintf(logBuf, sizeof(logBuf), "wal read error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, err, err, errStr, sysErr, sysErrStr); if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { - syncNodeEventLog(pData->pSyncNode, logBuf); + // syncNodeEventLog(pData->pSyncNode, logBuf); } else { syncNodeErrorLog(pData->pSyncNode, logBuf); } @@ -499,7 +499,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { snprintf(logBuf, sizeof(logBuf), "wal read error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, err, err, errStr, sysErr, sysErrStr); if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { - syncNodeEventLog(pData->pSyncNode, logBuf); + // syncNodeEventLog(pData->pSyncNode, logBuf); } else { syncNodeErrorLog(pData->pSyncNode, logBuf); } From e646c7304ca3a0ab2ac82b81b72e1a3699fdf562 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Tue, 19 Jul 2022 11:37:35 +0800 Subject: [PATCH 18/26] test: add debug case for TD-16209 --- tests/{pytest/insert => system-test/1-insert}/insert_drop.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{pytest/insert => system-test/1-insert}/insert_drop.py (100%) diff --git a/tests/pytest/insert/insert_drop.py b/tests/system-test/1-insert/insert_drop.py similarity index 100% rename from tests/pytest/insert/insert_drop.py rename to tests/system-test/1-insert/insert_drop.py From ca5e959a1e05fddf953e5e28d6451a5346ef5e80 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 19 Jul 2022 11:46:17 +0800 Subject: [PATCH 19/26] fix(query): fix derivative nan value TD-17511 --- source/libs/function/src/builtinsimpl.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 4f829f2df1..9d40b15354 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5661,7 +5661,12 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) { double r = ((v - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); if (pDerivInfo->ignoreNegative && r < 0) { } else { - colDataAppend(pOutput, pos, (const char*)&r, false); + if (isinf(r) || isnan(r)) { + colDataAppendNULL(pOutput, pos); + } else { + colDataAppend(pOutput, pos, (const char*)&r, false); + } + if (pTsOutput != NULL) { colDataAppendInt64(pTsOutput, pos, &tsList[i]); } @@ -5688,7 +5693,12 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) { double r = ((pDerivInfo->prevValue - v) * pDerivInfo->tsWindow) / (pDerivInfo->prevTs - tsList[i]); if (pDerivInfo->ignoreNegative && r < 0) { } else { - colDataAppend(pOutput, pos, (const char*)&r, false); + if (isinf(r) || isnan(r)) { + colDataAppendNULL(pOutput, pos); + } else { + colDataAppend(pOutput, pos, (const char*)&r, false); + } + if (pTsOutput != NULL) { colDataAppendInt64(pTsOutput, pos, &pDerivInfo->prevTs); } From 83ac069beff626be3bb0cadda37d43cfca571df9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 19 Jul 2022 12:00:43 +0800 Subject: [PATCH 20/26] enh: If an 'object is creating' error is reported when creating a database, wait for the transaction to end then response --- source/dnode/mnode/impl/inc/mndDef.h | 43 +++++++++- source/dnode/mnode/impl/inc/mndPrivilege.h | 40 --------- source/dnode/mnode/impl/inc/mndTrans.h | 2 + source/dnode/mnode/impl/src/mndDb.c | 9 ++ source/dnode/mnode/impl/src/mndTrans.c | 98 +++++++++++++++++----- tests/script/jenkins/basic.txt | 4 +- tests/script/tsim/valgrind/checkError1.sim | 2 +- 7 files changed, 131 insertions(+), 67 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 5cf6e26ee8..977f33de49 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -35,6 +35,46 @@ extern "C" { #endif +typedef enum { + MND_OPER_CONNECT = 1, + MND_OPER_CREATE_ACCT, + MND_OPER_DROP_ACCT, + MND_OPER_ALTER_ACCT, + MND_OPER_CREATE_USER, + MND_OPER_DROP_USER, + MND_OPER_ALTER_USER, + MND_OPER_CREATE_BNODE, + MND_OPER_DROP_BNODE, + MND_OPER_CREATE_DNODE, + MND_OPER_DROP_DNODE, + MND_OPER_CONFIG_DNODE, + MND_OPER_CREATE_MNODE, + MND_OPER_DROP_MNODE, + MND_OPER_CREATE_QNODE, + MND_OPER_DROP_QNODE, + MND_OPER_CREATE_SNODE, + MND_OPER_DROP_SNODE, + MND_OPER_REDISTRIBUTE_VGROUP, + MND_OPER_MERGE_VGROUP, + MND_OPER_SPLIT_VGROUP, + MND_OPER_BALANCE_VGROUP, + MND_OPER_CREATE_FUNC, + MND_OPER_DROP_FUNC, + MND_OPER_KILL_TRANS, + MND_OPER_KILL_CONN, + MND_OPER_KILL_QUERY, + MND_OPER_CREATE_DB, + MND_OPER_ALTER_DB, + MND_OPER_DROP_DB, + MND_OPER_COMPACT_DB, + MND_OPER_TRIM_DB, + MND_OPER_USE_DB, + MND_OPER_WRITE_DB, + MND_OPER_READ_DB, + MND_OPER_READ_OR_WRITE_DB, + MND_OPER_SHOW_VARIBALES, +} EOperType; + typedef enum { MND_AUTH_ACCT_START = 0, MND_AUTH_ACCT_USER, @@ -109,9 +149,9 @@ typedef struct { ETrnPolicy policy; ETrnConflct conflict; ETrnExec exec; + EOperType oper; int32_t code; int32_t failedTimes; - SRpcHandleInfo rpcInfo; void* rpcRsp; int32_t rpcRspLen; int32_t redoActionPos; @@ -130,6 +170,7 @@ typedef struct { int32_t stopFunc; int32_t paramLen; void* param; + SArray* pRpcArray; } STrans; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndPrivilege.h b/source/dnode/mnode/impl/inc/mndPrivilege.h index a149c0f0e2..f6002e3be8 100644 --- a/source/dnode/mnode/impl/inc/mndPrivilege.h +++ b/source/dnode/mnode/impl/inc/mndPrivilege.h @@ -22,46 +22,6 @@ extern "C" { #endif -typedef enum { - MND_OPER_CONNECT = 1, - MND_OPER_CREATE_ACCT, - MND_OPER_DROP_ACCT, - MND_OPER_ALTER_ACCT, - MND_OPER_CREATE_USER, - MND_OPER_DROP_USER, - MND_OPER_ALTER_USER, - MND_OPER_CREATE_BNODE, - MND_OPER_DROP_BNODE, - MND_OPER_CREATE_DNODE, - MND_OPER_DROP_DNODE, - MND_OPER_CONFIG_DNODE, - MND_OPER_CREATE_MNODE, - MND_OPER_DROP_MNODE, - MND_OPER_CREATE_QNODE, - MND_OPER_DROP_QNODE, - MND_OPER_CREATE_SNODE, - MND_OPER_DROP_SNODE, - MND_OPER_REDISTRIBUTE_VGROUP, - MND_OPER_MERGE_VGROUP, - MND_OPER_SPLIT_VGROUP, - MND_OPER_BALANCE_VGROUP, - MND_OPER_CREATE_FUNC, - MND_OPER_DROP_FUNC, - MND_OPER_KILL_TRANS, - MND_OPER_KILL_CONN, - MND_OPER_KILL_QUERY, - MND_OPER_CREATE_DB, - MND_OPER_ALTER_DB, - MND_OPER_DROP_DB, - MND_OPER_COMPACT_DB, - MND_OPER_TRIM_DB, - MND_OPER_USE_DB, - MND_OPER_WRITE_DB, - MND_OPER_READ_DB, - MND_OPER_READ_OR_WRITE_DB, - MND_OPER_SHOW_VARIBALES, -} EOperType; - int32_t mndInitPrivilege(SMnode *pMnode); void mndCleanupPrivilege(SMnode *pMnode); diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 6e367fbe24..faf656a251 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -73,12 +73,14 @@ void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *param, int32_t paramLen); void mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2); void mndTransSetSerial(STrans *pTrans); +void mndTransSetOper(STrans *pTrans, EOperType oper); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); int32_t mndTransProcessRsp(SRpcMsg *pRsp); void mndTransPullup(SMnode *pMnode); int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans); void mndTransExecute(SMnode *pMnode, STrans *pTrans); +int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, const char *dbname); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index b4b1b383e7..064ef9b40a 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -487,6 +487,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db); mndTransSetDbName(pTrans, dbObj.name, NULL); + mndTransSetOper(pTrans, MND_OPER_CREATE_DB); if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; @@ -534,6 +535,14 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { terrno = TSDB_CODE_MND_DB_ALREADY_EXIST; goto _OVER; } + } else if (terrno == TSDB_CODE_SDB_OBJ_CREATING) { + if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) { + mDebug("db:%s, is creating and response after trans finished", createReq.db); + code = TSDB_CODE_ACTION_IN_PROGRESS; + goto _OVER; + } else { + goto _OVER; + } } else if (terrno != TSDB_CODE_MND_DB_NOT_EXIST) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index ea92b2f0e6..679f8085bd 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -122,6 +122,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT8(pRaw, dataPos, pTrans->policy, _OVER) SDB_SET_INT8(pRaw, dataPos, pTrans->conflict, _OVER) SDB_SET_INT8(pRaw, dataPos, pTrans->exec, _OVER) + SDB_SET_INT8(pRaw, dataPos, pTrans->oper, _OVER) + SDB_SET_INT8(pRaw, dataPos, 0, _OVER) + SDB_SET_INT8(pRaw, dataPos, 0, _OVER) + SDB_SET_INT8(pRaw, dataPos, 0, _OVER) SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, _OVER) SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname1, TSDB_DB_FNAME_LEN, _OVER) SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname2, TSDB_DB_FNAME_LEN, _OVER) @@ -269,15 +273,22 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { int8_t policy = 0; int8_t conflict = 0; int8_t exec = 0; + int8_t oper = 0; + int8_t reserved = 0; int8_t actionType = 0; SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) SDB_GET_INT8(pRaw, dataPos, &policy, _OVER) SDB_GET_INT8(pRaw, dataPos, &conflict, _OVER) SDB_GET_INT8(pRaw, dataPos, &exec, _OVER) + SDB_GET_INT8(pRaw, dataPos, &oper, _OVER) + SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER) + SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER) + SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER) pTrans->stage = stage; pTrans->policy = policy; pTrans->conflict = conflict; pTrans->exec = exec; + pTrans->oper = oper; SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, _OVER) SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname1, TSDB_DB_FNAME_LEN, _OVER) SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname2, TSDB_DB_FNAME_LEN, _OVER) @@ -495,6 +506,10 @@ static void mndTransDropData(STrans *pTrans) { mndTransDropActions(pTrans->commitActions); pTrans->commitActions = NULL; } + if (pTrans->pRpcArray != NULL) { + taosArrayDestroy(pTrans->pRpcArray); + pTrans->pRpcArray = NULL; + } if (pTrans->rpcRsp != NULL) { taosMemoryFree(pTrans->rpcRsp); pTrans->rpcRsp = NULL; @@ -585,14 +600,18 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->commitActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); + pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo)); - if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL) { + if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL || + pTrans->pRpcArray == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to create transaction since %s", terrstr()); return NULL; } - if (pReq != NULL) pTrans->rpcInfo = pReq->info; + if (pReq != NULL) { + taosArrayPush(pTrans->pRpcArray, &pReq->info); + } mTrace("trans:%d, local object is created, data:%p", pTrans->id, pTrans); return pTrans; } @@ -677,6 +696,31 @@ void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void * pTrans->paramLen = paramLen; } +int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, const char *dbname) { + STrans *pTrans = NULL; + void *pIter = NULL; + int32_t code = -1; + + while (1) { + pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans); + if (pIter == NULL) break; + + if (pTrans->oper == oper) { + if (strcasecmp(dbname, pTrans->dbname1) == 0) { + mDebug("trans:%d, db:%s oper:%d matched with input", pTrans->id, dbname, oper); + if (taosArrayPush(pTrans->pRpcArray, &pMsg->info) != NULL) { + code = 0; + } + sdbRelease(pMnode->pSdb, pTrans); + break; + } + } + + sdbRelease(pMnode->pSdb, pTrans); + } + return code; +} + void mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2) { if (dbname1 != NULL) { memcpy(pTrans->dbname1, dbname1, TSDB_DB_FNAME_LEN); @@ -688,6 +732,8 @@ void mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2) void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; } +void mndTransSetOper(STrans *pTrans, EOperType oper) { pTrans->oper = oper; } + static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { SSdbRaw *pRaw = mndTransActionEncode(pTrans); if (pRaw == NULL) { @@ -711,7 +757,7 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { static bool mndCheckDbConflict(const char *db, STrans *pTrans) { if (db[0] == 0) return false; - if (strcmp(db, pTrans->dbname1) == 0 || strcmp(db, pTrans->dbname2) == 0) return true; + if (strcasecmp(db, pTrans->dbname1) == 0 || strcasecmp(db, pTrans->dbname2) == 0) return true; return false; } @@ -784,9 +830,10 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { return -1; } - pNew->rpcInfo = pTrans->rpcInfo; + pNew->pRpcArray = pTrans->pRpcArray; pNew->rpcRsp = pTrans->rpcRsp; pNew->rpcRspLen = pTrans->rpcRspLen; + pTrans->pRpcArray = NULL; pTrans->rpcRsp = NULL; pTrans->rpcRspLen = 0; @@ -835,29 +882,34 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { } } - if (sendRsp && pTrans->rpcInfo.handle != NULL) { - mDebug("trans:%d, send rsp, code:0x%x stage:%s app:%p", pTrans->id, code, mndTransStr(pTrans->stage), - pTrans->rpcInfo.ahandle); - if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL; - } - SRpcMsg rspMsg = {.code = code, .info = pTrans->rpcInfo}; + if (!sendRsp) return; - if (pTrans->rpcRspLen != 0) { - void *rpcCont = rpcMallocCont(pTrans->rpcRspLen); - if (rpcCont != NULL) { - memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen); - rspMsg.pCont = rpcCont; - rspMsg.contLen = pTrans->rpcRspLen; + int32_t size = taosArrayGetSize(pTrans->pRpcArray); + if (size <= 0) return; + + for (int32_t i = 0; i < size; ++i) { + SRpcHandleInfo *pInfo = taosArrayGet(pTrans->pRpcArray, i); + if (pInfo->handle != NULL) { + mDebug("trans:%d, send rsp, code:0x%x stage:%s app:%p", pTrans->id, code, mndTransStr(pTrans->stage), + pInfo->ahandle); + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL; } - taosMemoryFree(pTrans->rpcRsp); - } + SRpcMsg rspMsg = {.code = code, .info = *pInfo}; - tmsgSendRsp(&rspMsg); - pTrans->rpcInfo.handle = NULL; - pTrans->rpcRsp = NULL; - pTrans->rpcRspLen = 0; + if (pTrans->rpcRspLen != 0) { + void *rpcCont = rpcMallocCont(pTrans->rpcRspLen); + if (rpcCont != NULL) { + memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen); + rspMsg.pCont = rpcCont; + rspMsg.contLen = pTrans->rpcRspLen; + } + } + + tmsgSendRsp(&rspMsg); + } } + taosArrayClear(pTrans->pRpcArray); } int32_t mndTransProcessRsp(SRpcMsg *pRsp) { diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index bff6177ad2..1b228925b1 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -248,8 +248,8 @@ ./test.sh -f tsim/stream/sliding.sim # ---- transaction - ./test.sh -f tsim/trans/lossdata1.sim - ./test.sh -f tsim/trans/create_db.sim +./test.sh -f tsim/trans/lossdata1.sim +./test.sh -f tsim/trans/create_db.sim # ---- tmq ./test.sh -f tsim/tmq/basic1.sim diff --git a/tests/script/tsim/valgrind/checkError1.sim b/tests/script/tsim/valgrind/checkError1.sim index 76a29ee62f..59eede7c91 100644 --- a/tests/script/tsim/valgrind/checkError1.sim +++ b/tests/script/tsim/valgrind/checkError1.sim @@ -152,7 +152,7 @@ endi system_content sh/checkValgrind.sh -n dnode2 print cmd return result ----> [ $system_content ] -if $system_content > 4 then +if $system_content > 0 then return -1 endi From ce999711ec00bd4205fd319cd688e68ac98f6f8c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 19 Jul 2022 13:07:24 +0800 Subject: [PATCH 21/26] test: add case --- tests/script/tmp/data.sim | 125 ++++++++++++++++++++-- tests/script/tmp/monitor.sim | 21 +--- tests/script/tsim/db/alter_replica_31.sim | 9 -- 3 files changed, 121 insertions(+), 34 deletions(-) diff --git a/tests/script/tmp/data.sim b/tests/script/tmp/data.sim index 4e714c01ee..dcfa02e0a7 100644 --- a/tests/script/tmp/data.sim +++ b/tests/script/tmp/data.sim @@ -1,18 +1,125 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c telemetryReporting -v 1 -system sh/cfg.sh -n dnode1 -c telemetryInterval -v 1 -system sh/cfg.sh -n dnode1 -c telemetryServer -v localhost -system sh/cfg.sh -n dnode1 -c telemetryPort -v 80 - -return +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start sql connect -sql create database db -sql create table db.tb (ts timestamp, i int) -sql insert into db.tb values(now, 1) +print =============== step1: create dnodes +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> rows: $rows +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data24 $data35 +if $rows != 4 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +print =============== step2: create database +sql create database db vgroups 1 replica 3 +sql show databases +if $rows != 3 then + return -1 +endi +if $data(db)[4] != 3 then + return -1 +endi + +sql show dnodes +if $data(2)[2] != 1 then + return -1 +endi +if $data(3)[2] != 1 then + return -1 +endi +if $data(4)[2] != 1 then + return -1 +endi + +# vnodes +sql show dnodes +if $data(2)[2] != 1 then + return -1 +endi +if $data(3)[2] != 1 then + return -1 +endi +if $data(4)[2] != 1 then + return -1 +endi + +# v1_dnode +$hasleader = 0 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 20 then + print ====> dnode not ready! + return -1 + endi +sql show db.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 +if $data(2)[3] != 2 then + return -1 +endi +if $data(2)[5] != 3 then + return -1 +endi +if $data(2)[7] != 4 then + return -1 +endi +if $data(2)[4] == leader then + $hasleader = 1 +endi +if $data(2)[6] == leader then + $hasleader = 1 +endi +if $data(2)[8] == leader then + $hasleader = 1 +endi +if $hasleader != 1 then + goto step2 +endi + +sql use db; +sql create table stb (ts timestamp, c int) tags (t int); +sql create table t0 using stb tags (0); +sql insert into t0 values(now, 1); +sql show db.stables; +sql show db.tables; +sql show db.vgroups; + +return print ======== start back run_back tmp/back.sim diff --git a/tests/script/tmp/monitor.sim b/tests/script/tmp/monitor.sim index 036d4cc6db..ec728fe733 100644 --- a/tests/script/tmp/monitor.sim +++ b/tests/script/tmp/monitor.sim @@ -6,32 +6,21 @@ system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 system sh/cfg.sh -n dnode1 -c monitorComp -v 1 #system sh/cfg.sh -n dnode1 -c supportVnodes -v 128 +#system sh/cfg.sh -n dnode1 -c telemetryReporting -v 1 +#system sh/cfg.sh -n dnode1 -c telemetryInterval -v 1 +#system sh/cfg.sh -n dnode1 -c telemetryServer -v localhost +#system sh/cfg.sh -n dnode1 -c telemetryPort -v 80 + system sh/exec.sh -n dnode1 -s start sql connect print =============== show dnodes -sleep 2000 sql create database db vgroups 2; sql use db; sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd"; -sleep 2000 print =============== create drop qnode 1 sql create qnode on dnode 1 sql create snode on dnode 1 sql create bnode on dnode 1 -return -print =============== restart -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode1 -s start - - -return -system sh/deploy.sh -n dnode2 -i 2 -system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s start - -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/db/alter_replica_31.sim b/tests/script/tsim/db/alter_replica_31.sim index 1823f182c9..e9a295820c 100644 --- a/tests/script/tsim/db/alter_replica_31.sim +++ b/tests/script/tsim/db/alter_replica_31.sim @@ -111,15 +111,6 @@ if $hasleader != 1 then goto step2 endi -# sql use db; -# sql create table stb (ts timestamp, c int) tags (t int); -# sql create table t0 using stb tags (0); -# sql insert into t0 values(now, 1); -# sql show db.stables; -# sql show db.tables; -# sql show db.vgroups; -return - sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd" sql create table db.ctb using db.stb tags(101, "102") sql insert into db.ctb values(now, 1, "2") From 007c5eb0a510e1cd747a224087a43e536e609f61 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 19 Jul 2022 13:27:00 +0800 Subject: [PATCH 22/26] chore: libtaos ws submodule for3.0 (#15105) * chore: add libtaos-ws for 3.0 * chore: update taosws-rs * chore: add libtaosws to install/remove script * chore: update taosws-rs * chore: update taosws-rs * chore: update taos-tools, taosws-rs for 3.0 * fix: packaging/tools/make_install.sh for 3.0 * chore: update taos-tools * chore: fix release script for 3.0 * chore: update taosws-rs for 3.0 * chore: add taows-rs submodule for 3.0 * chore: update taosws-rs for 3.0 * fix: install script support taosws for 3.0 --- packaging/deb/makedeb.sh | 2 +- packaging/rpm/tdengine.spec | 2 +- packaging/tools/install.sh | 4 ++-- packaging/tools/install_client.sh | 12 ++++++++++++ packaging/tools/makeclient.sh | 7 +++++++ packaging/tools/makepkg.sh | 4 ++-- packaging/tools/post.sh | 11 +++++++++++ 7 files changed, 36 insertions(+), 6 deletions(-) diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 1a4131ec6f..f09d46da59 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -73,7 +73,7 @@ cp ${compile_dir}/../include/client/taos.h ${pkg_dir}${install_home_pat cp ${compile_dir}/../include/common/taosdef.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/util/taoserror.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/libs/function/taosudf.h ${pkg_dir}${install_home_path}/include -cp ${compile_dir}/../src/inc/taosws.h ${pkg_dir}${install_home_path}/include ||: +cp ${compile_dir}/build/include/taosws.h ${pkg_dir}${install_home_path}/include ||: cp -r ${top_dir}/examples/* ${pkg_dir}${install_home_path}/examples #cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector #cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 02e7e9b5e5..bf0921f18a 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -80,7 +80,7 @@ cp %{_compiledir}/../include/client/taos.h %{buildroot}%{homepath}/incl cp %{_compiledir}/../include/common/taosdef.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/util/taoserror.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/libs/function/taosudf.h %{buildroot}%{homepath}/include -cp %{_compiledir}/../src/inc/taosws.h %{buildroot}%{homepath}/include ||: +cp %{_compiledir}/build/include/taosws.h %{buildroot}%{homepath}/include ||: #cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector #cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector #cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index f409834091..72ac76ec8e 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -229,13 +229,13 @@ function install_lib() { ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - ${csudo}ln -s ${lib_link_dir}/libtaosws.so ${lib_link_dir}/libtaosws.so || : + [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || : if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : - ${csudo}ln -s ${lib64_link_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : + ${csudo}ln -s ${install_main_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : fi ${csudo}ldconfig diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 0c86877c99..a205206dda 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -116,6 +116,7 @@ function install_bin() { function clean_lib() { sudo rm -f /usr/lib/libtaos.* || : + sudo rm -f /usr/lib/libtaosws.* || : sudo rm -rf ${lib_dir} || : } @@ -123,6 +124,9 @@ function install_lib() { # Remove links ${csudo}rm -f ${lib_link_dir}/libtaos.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + + ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : + ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : #${csudo}rm -rf ${v15_java_app_dir} || : ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* @@ -131,13 +135,19 @@ function install_lib() { ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so + if [ -d "${lib64_link_dir}" ]; then ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + + [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.so ${lib64_link_dir}/libtaosws.so || : fi else ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib ${csudo}ln -s ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib + + [ -f ${install_main_dir}/driver/libtaosws.dylib ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.dylib ${lib_link_dir}/libtaosws.dylib fi if [ "$osType" != "Darwin" ]; then @@ -154,6 +164,8 @@ function install_header() { ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h + + [ -f ${install_main_dir}/include/taosws.h ] && ${csudo}ln -s ${install_main_dir}/include/taosws.h ${inc_link_dir}/taos.h } function install_jemalloc() { diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index ca3a9a19be..ae846ee493 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -57,12 +57,16 @@ if [ "$osType" != "Darwin" ]; then ${script_dir}/get_client.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" + wslib_files="${build_dir}/lib/libtaosws.so" else bin_files="${build_dir}/bin/${clientName} ${script_dir}/remove_client.sh" lib_files="${build_dir}/lib/libtaos.${version}.dylib" + wslib_files="${build_dir}/lib/libtaosws.dylib" fi header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h" +wsheader_files="${build_dir}/include/taosws.h" + if [ "$dbName" != "taos" ]; then cfg_dir="${top_dir}/../enterprise/packaging/cfg" else @@ -74,6 +78,8 @@ install_files="${script_dir}/install_client.sh" # make directories. mkdir -p ${install_dir} mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc +[ -f ${wsheader_files} ] && cp ${wsheader_files} ${install_dir}/inc + mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile} mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* @@ -187,6 +193,7 @@ cp ${lib_files} ${install_dir}/driver # Copy connector connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector +[ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index c37397005c..f34adca9f8 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -96,7 +96,7 @@ lib_files="${build_dir}/lib/libtaos.so.${version}" wslib_files="${build_dir}/lib/libtaosws.so." header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h" -wsheader_files="${code_dir}/inc/taosws.h" +wsheader_files="${build_dir}/include/taosws.h" if [ "$dbName" != "taos" ]; then cfg_dir="${top_dir}/../enterprise/packaging/cfg" @@ -114,7 +114,7 @@ init_file_rpm=${script_dir}/../rpm/taosd mkdir -p ${install_dir} mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc -${wsheader_files} ${install_dir}/inc || : +[ -f ${wsheader_files} ] && cp ${wsheader_files} ${install_dir}/inc || : mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile} diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 2d744233ba..28163775a1 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -82,22 +82,33 @@ function kill_taosd() { function install_include() { ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || : + ${csudo}rm -f ${inc_link_dir}/taosws.h + ${csudo}ln -s ${inc_dir}/taos.h ${inc_link_dir}/taos.h ${csudo}ln -s ${inc_dir}/taosdef.h ${inc_link_dir}/taosdef.h ${csudo}ln -s ${inc_dir}/taoserror.h ${inc_link_dir}/taoserror.h ${csudo}ln -s ${inc_dir}/taosudf.h ${inc_link_dir}/taosudf.h + + [ -f ${inc_dir}/taosws.h ] && ${csudo}ln -s ${inc_dir}/taosudf.h ${inc_link_dir}/taosudf.h ||: } function install_lib() { ${csudo}rm -f ${lib_link_dir}/libtaos* || : ${csudo}rm -f ${lib64_link_dir}/libtaos* || : + ${csudo}rm -f ${lib_link_dir}/libtaosws* || : + ${csudo}rm -f ${lib64_link_dir}/libtaosws* || : + ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + [ -f ${lib_dir}/libtaosws.so ]${csudo}ln -s ${lib_dir}/libtaosws.so ${lib_link_dir}/libtaosws.so + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + + [ -ff ${lib_dir}/libtaosws.so ] && ${csudo}ln -s ${lib_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : fi ${csudo}ldconfig From fc08a35fbff9eef43d4a68c0a51eba5ac9667f84 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 19 Jul 2022 13:52:02 +0800 Subject: [PATCH 23/26] refactor(sync): update sync-timeout to 10s --- include/libs/sync/sync.h | 2 +- source/libs/sync/src/syncMain.c | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 06a58af0e3..510f816959 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -26,7 +26,7 @@ extern "C" { extern bool gRaftDetailLog; -#define SYNC_RESP_TTL_MS 5000 +#define SYNC_RESP_TTL_MS 10000 #define SYNC_MAX_BATCH_SIZE 500 #define SYNC_INDEX_BEGIN 0 diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index ddb2b9355e..e11f610dd4 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1552,7 +1552,8 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { } else { snprintf(logBuf, sizeof(logBuf), "%s", str); } - sDebug("%s", logBuf); + // sDebug("%s", logBuf); + sInfo("%s", logBuf); } else { int len = 256 + userStrLen; @@ -1573,7 +1574,8 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { } else { snprintf(s, len, "%s", str); } - sDebug("%s", s); + // sDebug("%s", s); + sInfo("%s", s); taosMemoryFree(s); } From 8e41fc912be8da01d69d9272f91a982dcc61729b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 19 Jul 2022 14:17:53 +0800 Subject: [PATCH 24/26] feat(stream): add num of children --- include/libs/executor/executor.h | 1 + include/libs/stream/tstream.h | 1 + source/dnode/mnode/impl/src/mndScheduler.c | 5 +++++ source/dnode/vnode/src/tq/tq.c | 6 +++++- source/libs/executor/src/executorimpl.c | 4 ++-- source/libs/executor/src/scanoperator.c | 2 +- source/libs/stream/src/streamTask.c | 2 ++ 7 files changed, 17 insertions(+), 4 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 8b0a836ad2..9e28911784 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -40,6 +40,7 @@ typedef struct SReadHandle { bool initMetaReader; bool initTableReader; bool initTqReader; + int32_t numOfVgroups; } SReadHandle; // in queue mode, data streams are seperated by msg diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 8c69c0f2de..a484492d08 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -262,6 +262,7 @@ typedef struct SStreamTask { int64_t startVer; int64_t checkpointVer; int64_t processedVer; + int32_t numOfVgroups; // children info SArray* childEpInfo; // SArray diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index ec0ea90f46..a46938590e 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -383,6 +383,11 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { // exec pInnerTask->execType = TASK_EXEC__PIPE; + SDbObj* pSourceDb = mndAcquireDb(pMnode, pStream->sourceDb); + ASSERT(pDbObj != NULL); + sdbRelease(pSdb, pSourceDb); + pInnerTask->numOfVgroups = pSourceDb->cfg.numOfVgroups; + if (tsSchedStreamToSnode) { SSnodeObj* pSnode = mndSchedFetchOneSnode(pMnode); if (pSnode == NULL) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index fb05aeecd9..1802f9be1a 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -591,7 +591,11 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { }; pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); } else { - pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL); + SReadHandle mgHandle = { + .vnode = NULL, + .numOfVgroups = pTask->numOfVgroups, + }; + pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle); } ASSERT(pTask->exec.executor); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 06bb096e59..774de10649 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4424,7 +4424,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN, }; - if (pHandle) { + if (pHandle->vnode) { int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo)); if (code) { @@ -4590,7 +4590,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t children = 0; pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL == type) { - int32_t children = 1; + int32_t children = pHandle->numOfVgroups; pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) { pOptr = createSortOperatorInfo(ops[0], (SSortPhysiNode*)pPhyNode, pTaskInfo); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e60f6f8a5b..57ba08e8fa 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1539,7 +1539,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys goto _error; } - if (pHandle) { + if (pHandle->vnode) { SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); STableScanInfo* pTSInfo = (STableScanInfo*)pTableScanOp->info; if (pHandle->version > 0) { diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 2d15c31bf1..5921e44a9c 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -64,6 +64,7 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { if (tEncodeI32(pEncoder, pTask->selfChildId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->numOfVgroups) < 0) return -1; int32_t epSz = taosArrayGetSize(pTask->childEpInfo); if (tEncodeI32(pEncoder, epSz) < 0) return -1; @@ -118,6 +119,7 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { if (tDecodeI32(pDecoder, &pTask->selfChildId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->numOfVgroups) < 0) return -1; int32_t epSz; if (tDecodeI32(pDecoder, &epSz) < 0) return -1; From 99fd74c74715b595f7e048d4f91b160d5c31a02f Mon Sep 17 00:00:00 2001 From: huolibo Date: Tue, 19 Jul 2022 10:01:56 +0800 Subject: [PATCH 25/26] docs(driver): release jdbc 2.0.40 --- docs/en/14-reference/03-connector/java.mdx | 23 +++++++++++++++------- docs/zh/14-reference/03-connector/java.mdx | 11 ++++++++++- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/docs/en/14-reference/03-connector/java.mdx b/docs/en/14-reference/03-connector/java.mdx index ff15acf1a9..625ffc2698 100644 --- a/docs/en/14-reference/03-connector/java.mdx +++ b/docs/en/14-reference/03-connector/java.mdx @@ -202,6 +202,10 @@ The configuration parameters in the URL are as follows. - batchfetch: true: pull the result set in batch when executing the query; false: pull the result set row by row. The default value is false. batchfetch uses HTTP for data transfer. The JDBC REST connection supports bulk data pulling function in taos-jdbcdriver-2.0.38 and TDengine 2.4.0.12 and later versions. taos-jdbcdriver and TDengine transfer data via WebSocket connection. Compared with HTTP, WebSocket enables JDBC REST connection to support large data volume querying and improve query performance. - charset: specify the charset to parse the string, this parameter is valid only when set batchfetch to true. - batchErrorIgnore: true: when executing executeBatch of Statement, if one SQL execution fails in the middle, continue to execute the following SQL. false: no longer execute any statement after the failed SQL. The default value is: false. +- httpConnectTimeout: REST connection timeout in milliseconds, the default value is 5000 ms. +- httpSocketTimeout: socket timeout in milliseconds, the default value is 5000 ms. only takes effect when batchfetch is false. +- messageWaitTimeout: message transmission timeout in milliseconds, the default value is 3000 ms. only takes effect when batchfetch is true. +- useSSL: connecting Securely Using SSL. true: using SSL conneciton, false: not using SSL connection. **Note**: Some configuration items (e.g., locale, timezone) do not work in the REST connection. @@ -257,14 +261,18 @@ In the above example, a connection is established to `taosdemo.com`, port is 603 The configuration parameters in properties are as follows. -- TSDBDriver.PROPERTY_KEY_USER: Login TDengine user name, default value 'root'. +- TSDBDriver.PROPERTY_KEY_USER: login TDengine user name, default value 'root'. - TSDBDriver.PROPERTY_KEY_PASSWORD: user login password, default value 'taosdata'. - TSDBDriver.PROPERTY_KEY_BATCH_LOAD: true: pull the result set in batch when executing query; false: pull the result set row by row. The default value is: false. - TSDBDriver.PROPERTY_KEY_BATCH_ERROR_IGNORE: true: when executing executeBatch of Statement, if there is a SQL execution failure in the middle, continue to execute the following sq. false: no longer execute any statement after the failed SQL. The default value is: false. -- TSDBDriver.PROPERTY_KEY_CONFIG_DIR: Only works when using JDBC native connection. Client configuration file directory path, default value `/etc/taos` on Linux OS, default value `C:/TDengine/cfg` on Windows OS. +- TSDBDriver.PROPERTY_KEY_CONFIG_DIR: only works when using JDBC native connection. Client configuration file directory path, default value `/etc/taos` on Linux OS, default value `C:/TDengine/cfg` on Windows OS. - TSDBDriver.PROPERTY_KEY_CHARSET: In the character set used by the client, the default value is the system character set. - TSDBDriver.PROPERTY_KEY_LOCALE: this only takes effect when using JDBC native connection. Client language environment, the default value is system current locale. - TSDBDriver.PROPERTY_KEY_TIME_ZONE: only takes effect when using JDBC native connection. In the time zone used by the client, the default value is the system's current time zone. +- TSDBDriver.HTTP_CONNECT_TIMEOUT: REST connection timeout in milliseconds, the default value is 5000 ms. only takes effect when using JDBC REST connection. +- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket timeout in milliseconds, the default value is 5000 ms. only takes effect when using JDBC REST connection and batchfetch is false. +- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 3000 ms. only takes effect when using JDBC REST connection and batchfetch is true. +- TSDBDriver.PROPERTY_KEY_USE_SSL: connecting Securely Using SSL. true: using SSL conneciton, false: not using SSL connection. only takes effect when using using JDBC REST connection. For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](/reference/config/#Client-Only). ### Priority of configuration parameters @@ -812,11 +820,12 @@ Please refer to: [JDBC example](https://github.com/taosdata/TDengine/tree/develo ## Recent update logs -| taos-jdbcdriver version | major changes | -| :---------------------: | :------------------------------------------: | -| 2.0.38 | JDBC REST connections add bulk pull function | -| 2.0.37 | Added support for json tags | -| 2.0.36 | Add support for schemaless writing | +| taos-jdbcdriver version | major changes | +| :---------------------: | :--------------------------------------------: | +| 2.0.39 - 2.0.40 | Add REST connection/request timeout parameters | +| 2.0.38 | JDBC REST connections add bulk pull function | +| 2.0.37 | Support json tags | +| 2.0.36 | Support schemaless writing | ## Frequently Asked Questions diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index ddab9e5f24..838fa2eff8 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -201,6 +201,10 @@ url 中的配置参数如下： - batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。从 taos-jdbcdriver-2.0.38 和 TDengine 2.4.0.12 版本开始，JDBC REST 连接增加批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。 - charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。 - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。 +- httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 5000。 +- httpSocketTimeout: socket 超时时间，单位 ms，默认值为 5000。仅在 batchfetch 设置为 false 时生效。 +- messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 batchfetch 设置为 true 时生效。 +- useSSL: 连接中是否使用 SSL。 **注意**：部分配置项（比如：locale、timezone）在 REST 连接中不生效。 @@ -264,7 +268,11 @@ properties 中的配置参数如下： - TSDBDriver.PROPERTY_KEY_CHARSET：客户端使用的字符集，默认值为系统字符集。 - TSDBDriver.PROPERTY_KEY_LOCALE：仅在使用 JDBC 原生连接时生效。 客户端语言环境，默认值系统当前 locale。 - TSDBDriver.PROPERTY_KEY_TIME_ZONE：仅在使用 JDBC 原生连接时生效。 客户端使用的时区，默认值为系统当前时区。 -- 此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。更多详细配置请参考[客户端配置](/reference/config/#仅客户端适用)。 +- TSDBDriver.HTTP_CONNECT_TIMEOUT: 连接超时时间，单位 ms， 默认值为 5000。仅在 REST 连接时生效。 +- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket 超时时间，单位 ms，默认值为 5000。仅在 REST 连接且 batchfetch 设置为 false 时生效。 +- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 REST 连接且 batchfetch 设置为 true 时生效。 +- TSDBDriver.PROPERTY_KEY_USE_SSL: 连接中是否使用 SSL。仅在 REST 连接时生效。 + 此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。更多详细配置请参考[客户端配置](/reference/config/#仅客户端适用)。 ### 配置参数的优先级 @@ -809,6 +817,7 @@ Query OK, 1 row(s) in set (0.000141s) | taos-jdbcdriver 版本 | 主要变化 | | :------------------: | :----------------------------: | +| 2.0.39 - 2.0.40 | 增加 REST 连接/请求 超时设置 | | 2.0.38 | JDBC REST 连接增加批量拉取功能 | | 2.0.37 | 增加对 json tag 支持 | | 2.0.36 | 增加对 schemaless 写入支持 | From 4a7d48418542b8998f64eea0922d91aac6113b2a Mon Sep 17 00:00:00 2001 From: huolibo Date: Tue, 19 Jul 2022 14:27:17 +0800 Subject: [PATCH 26/26] docs(driver): improve description --- docs/en/14-reference/03-connector/java.mdx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/14-reference/03-connector/java.mdx b/docs/en/14-reference/03-connector/java.mdx index 625ffc2698..310e0a15c6 100644 --- a/docs/en/14-reference/03-connector/java.mdx +++ b/docs/en/14-reference/03-connector/java.mdx @@ -203,8 +203,8 @@ The configuration parameters in the URL are as follows. - charset: specify the charset to parse the string, this parameter is valid only when set batchfetch to true. - batchErrorIgnore: true: when executing executeBatch of Statement, if one SQL execution fails in the middle, continue to execute the following SQL. false: no longer execute any statement after the failed SQL. The default value is: false. - httpConnectTimeout: REST connection timeout in milliseconds, the default value is 5000 ms. -- httpSocketTimeout: socket timeout in milliseconds, the default value is 5000 ms. only takes effect when batchfetch is false. -- messageWaitTimeout: message transmission timeout in milliseconds, the default value is 3000 ms. only takes effect when batchfetch is true. +- httpSocketTimeout: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when batchfetch is false. +- messageWaitTimeout: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when batchfetch is true. - useSSL: connecting Securely Using SSL. true: using SSL conneciton, false: not using SSL connection. **Note**: Some configuration items (e.g., locale, timezone) do not work in the REST connection. @@ -269,10 +269,10 @@ The configuration parameters in properties are as follows. - TSDBDriver.PROPERTY_KEY_CHARSET: In the character set used by the client, the default value is the system character set. - TSDBDriver.PROPERTY_KEY_LOCALE: this only takes effect when using JDBC native connection. Client language environment, the default value is system current locale. - TSDBDriver.PROPERTY_KEY_TIME_ZONE: only takes effect when using JDBC native connection. In the time zone used by the client, the default value is the system's current time zone. -- TSDBDriver.HTTP_CONNECT_TIMEOUT: REST connection timeout in milliseconds, the default value is 5000 ms. only takes effect when using JDBC REST connection. -- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket timeout in milliseconds, the default value is 5000 ms. only takes effect when using JDBC REST connection and batchfetch is false. -- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 3000 ms. only takes effect when using JDBC REST connection and batchfetch is true. -- TSDBDriver.PROPERTY_KEY_USE_SSL: connecting Securely Using SSL. true: using SSL conneciton, false: not using SSL connection. only takes effect when using using JDBC REST connection. +- TSDBDriver.HTTP_CONNECT_TIMEOUT: REST connection timeout in milliseconds, the default value is 5000 ms. It only takes effect when using JDBC REST connection. +- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when using JDBC REST connection and batchfetch is false. +- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when using JDBC REST connection and batchfetch is true. +- TSDBDriver.PROPERTY_KEY_USE_SSL: connecting Securely Using SSL. true: using SSL conneciton, false: not using SSL connection. It only takes effect when using using JDBC REST connection. For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](/reference/config/#Client-Only). ### Priority of configuration parameters