From 65f3a7ef8dfb3d5cfc9d5242aa25c43b9c02d9dd Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 30 Jun 2020 11:26:10 +0000 Subject: [PATCH 01/18] change the rpcMsg definition --- src/client/src/tscServer.c | 7 ++++--- src/inc/trpc.h | 8 ++++---- src/mnode/src/mnodeDnode.c | 2 +- src/mnode/src/mnodeTable.c | 18 +++++++++--------- src/mnode/src/mnodeVgroup.c | 18 +++++++++--------- src/rpc/src/rpcMain.c | 7 ++++--- src/rpc/test/rclient.c | 6 +++--- 7 files changed, 34 insertions(+), 32 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 470a829805..0ffe50c8f4 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -191,7 +191,8 @@ int tscSendMsgToServer(SSqlObj *pSql) { .msgType = pSql->cmd.msgType, .pCont = pMsg, .contLen = pSql->cmd.payloadLen, - .handle = pSql, + .ahandle = pSql, + .handle = &pSql->pRpcCtx, .code = 0 }; @@ -199,12 +200,12 @@ int tscSendMsgToServer(SSqlObj *pSql) { // Otherwise, the pSql object may have been released already during the response function, which is // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely // cause crash. - /*pSql->pRpcCtx = */rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg); + rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg); return TSDB_CODE_SUCCESS; } void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { - SSqlObj *pSql = (SSqlObj *)rpcMsg->handle; + SSqlObj *pSql = (SSqlObj *)rpcMsg->ahandle; if (pSql == NULL || pSql->signature != pSql) { tscError("%p sql is already released", pSql); return; diff --git a/src/inc/trpc.h b/src/inc/trpc.h index 30e0f9eee1..59362d669c 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -47,8 +47,8 @@ typedef struct SRpcMsg { void *pCont; int contLen; int32_t code; - void *handle; - void *ahandle; //app handle set by client, for debug purpose + void *handle; // rpc handle returned to app + void *ahandle; // app handle set by client } SRpcMsg; typedef struct SRpcInit { @@ -78,11 +78,11 @@ void rpcClose(void *); void *rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); void *rpcReallocCont(void *ptr, int contLen); -void *rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg); +void *rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg); void rpcSendResponse(const SRpcMsg *pMsg); void rpcSendRedirectRsp(void *pConn, const SRpcIpSet *pIpSet); int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); -void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, const SRpcMsg *pReq, SRpcMsg *pRsp); +void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pReq, SRpcMsg *pRsp); int rpcReportProgress(void *pConn, char *pCont, int contLen); void rpcCancelRequest(void *pContext); diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index dbe9185d8e..7e7c12cf08 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -264,7 +264,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { strcpy(pMdCfgDnode->config, pCmCfgDnode->config); SRpcMsg rpcMdCfgDnodeMsg = { - .handle = 0, + .ahandle = 0, .code = 0, .msgType = TSDB_MSG_TYPE_MD_CONFIG_DNODE, .pCont = pMdCfgDnode, diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 7dc0c5dfa5..1f1035887c 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1573,7 +1573,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup); SRpcMsg rpcMsg = { - .handle = pMsg, + .ahandle = pMsg, .pCont = pMDCreate, .contLen = htonl(pMDCreate->contLen), .code = 0, @@ -1750,7 +1750,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { mInfo("app:%p:%p, table:%s, send drop ctable msg", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); SRpcMsg rpcMsg = { - .handle = pMsg, + .ahandle = pMsg, .pCont = pDrop, .contLen = sizeof(SMDDropTableMsg), .code = 0, @@ -1798,7 +1798,7 @@ static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) { SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup); SRpcMsg rpcMsg = { - .handle = pMsg, + .ahandle = pMsg, .pCont = pMDCreate, .contLen = htonl(pMDCreate->contLen), .code = 0, @@ -2137,9 +2137,9 @@ static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) { // handle drop child response static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { - if (rpcMsg->handle == NULL) return; + if (rpcMsg->ahandle == NULL) return; - SMnodeMsg *mnodeMsg = rpcMsg->handle; + SMnodeMsg *mnodeMsg = rpcMsg->ahandle; mnodeMsg->received++; SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; @@ -2188,9 +2188,9 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { * if failed, drop the table cached */ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { - if (rpcMsg->handle == NULL) return; + if (rpcMsg->ahandle == NULL) return; - SMnodeMsg *mnodeMsg = rpcMsg->handle; + SMnodeMsg *mnodeMsg = rpcMsg->ahandle; mnodeMsg->received++; SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; @@ -2231,9 +2231,9 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { } static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) { - if (rpcMsg->handle == NULL) return; + if (rpcMsg->ahandle == NULL) return; - SMnodeMsg *mnodeMsg = rpcMsg->handle; + SMnodeMsg *mnodeMsg = rpcMsg->ahandle; mnodeMsg->received++; SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index b314bfc8d0..3855de4101 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -652,7 +652,7 @@ SRpcIpSet mnodeGetIpSetFromIp(char *ep) { void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) { SMDCreateVnodeMsg *pCreate = mnodeBuildCreateVnodeMsg(pVgroup); SRpcMsg rpcMsg = { - .handle = ahandle, + .ahandle = ahandle, .pCont = pCreate, .contLen = pCreate ? sizeof(SMDCreateVnodeMsg) : 0, .code = 0, @@ -673,9 +673,9 @@ void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) { } static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { - if (rpcMsg->handle == NULL) return; + if (rpcMsg->ahandle == NULL) return; - SMnodeMsg *mnodeMsg = rpcMsg->handle; + SMnodeMsg *mnodeMsg = rpcMsg->ahandle; mnodeMsg->received++; if (rpcMsg->code == TSDB_CODE_SUCCESS) { mnodeMsg->successed++; @@ -686,7 +686,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { SVgObj *pVgroup = mnodeMsg->pVgroup; mDebug("vgId:%d, create vnode rsp received, result:%s received:%d successed:%d expected:%d, thandle:%p ahandle:%p", pVgroup->vgId, tstrerror(rpcMsg->code), mnodeMsg->received, mnodeMsg->successed, mnodeMsg->expected, - mnodeMsg->rpcMsg.handle, rpcMsg->handle); + mnodeMsg->rpcMsg.handle, rpcMsg->ahandle); if (mnodeMsg->received != mnodeMsg->expected) return; @@ -718,7 +718,7 @@ static SMDDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) { void mnodeSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) { SMDDropVnodeMsg *pDrop = mnodeBuildDropVnodeMsg(vgId); SRpcMsg rpcMsg = { - .handle = ahandle, + .ahandle = ahandle, .pCont = pDrop, .contLen = pDrop ? sizeof(SMDDropVnodeMsg) : 0, .code = 0, @@ -737,10 +737,10 @@ static void mnodeSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) { } static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg) { - mDebug("drop vnode rsp is received, handle:%p", rpcMsg->handle); - if (rpcMsg->handle == NULL) return; + mDebug("drop vnode rsp is received, handle:%p", rpcMsg->ahandle); + if (rpcMsg->ahandle == NULL) return; - SMnodeMsg *mnodeMsg = rpcMsg->handle; + SMnodeMsg *mnodeMsg = rpcMsg->ahandle; mnodeMsg->received++; if (rpcMsg->code == TSDB_CODE_SUCCESS) { mnodeMsg->code = rpcMsg->code; @@ -750,7 +750,7 @@ static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg) { SVgObj *pVgroup = mnodeMsg->pVgroup; mDebug("vgId:%d, drop vnode rsp received, result:%s received:%d successed:%d expected:%d, thandle:%p ahandle:%p", pVgroup->vgId, tstrerror(rpcMsg->code), mnodeMsg->received, mnodeMsg->successed, mnodeMsg->expected, - mnodeMsg->rpcMsg.handle, rpcMsg->handle); + mnodeMsg->rpcMsg.handle, rpcMsg->ahandle); if (mnodeMsg->received != mnodeMsg->expected) return; diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index a59024f8db..26ea93416d 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -354,13 +354,13 @@ void *rpcReallocCont(void *ptr, int contLen) { return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); } -void *rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg) { +void *rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg) { SRpcInfo *pRpc = (SRpcInfo *)shandle; SRpcReqContext *pContext; int contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); pContext = (SRpcReqContext *) (pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); - pContext->ahandle = pMsg->handle; + pContext->ahandle = pMsg->ahandle; pContext->pRpc = (SRpcInfo *)shandle; pContext->ipSet = *pIpSet; pContext->contLen = contLen; @@ -380,6 +380,7 @@ void *rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg || type == TSDB_MSG_TYPE_CM_SHOW ) pContext->connType = RPC_CONN_TCPC; + if (pMsg->handle) *((void **)pMsg->handle) = pContext; rpcSendReqToServer(pRpc, pContext); return pContext; @@ -483,7 +484,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { return 0; } -void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, const SRpcMsg *pMsg, SRpcMsg *pRsp) { +void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { SRpcReqContext *pContext; pContext = (SRpcReqContext *) (pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); diff --git a/src/rpc/test/rclient.c b/src/rpc/test/rclient.c index 1bc64b0825..e51b54e299 100644 --- a/src/rpc/test/rclient.c +++ b/src/rpc/test/rclient.c @@ -33,7 +33,7 @@ typedef struct { } SInfo; static void processResponse(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { - SInfo *pInfo = (SInfo *)pMsg->handle; + SInfo *pInfo = (SInfo *)pMsg->ahandle; tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, pMsg->code); if (pIpSet) pInfo->ipSet = *pIpSet; @@ -46,7 +46,7 @@ static int tcount = 0; static void *sendRequest(void *param) { SInfo *pInfo = (SInfo *)param; - SRpcMsg rpcMsg; + SRpcMsg rpcMsg = {0}; tDebug("thread:%d, start to send request", pInfo->index); @@ -54,7 +54,7 @@ static void *sendRequest(void *param) { pInfo->num++; rpcMsg.pCont = rpcMallocCont(pInfo->msgSize); rpcMsg.contLen = pInfo->msgSize; - rpcMsg.handle = pInfo; + rpcMsg.ahandle = pInfo; rpcMsg.msgType = 1; tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); rpcSendRequest(pInfo->pRpc, &pInfo->ipSet, &rpcMsg); From 10eabebfbcd495701478337a012f46423b94ba2a Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 30 Jun 2020 11:57:08 +0000 Subject: [PATCH 02/18] change the return type --- src/inc/trpc.h | 2 +- src/rpc/src/rpcMain.c | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/inc/trpc.h b/src/inc/trpc.h index 59362d669c..d1adfb7494 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -78,7 +78,7 @@ void rpcClose(void *); void *rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); void *rpcReallocCont(void *ptr, int contLen); -void *rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg); +void rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg); void rpcSendResponse(const SRpcMsg *pMsg); void rpcSendRedirectRsp(void *pConn, const SRpcIpSet *pIpSet); int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 26ea93416d..dbdfa28a1a 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -354,7 +354,7 @@ void *rpcReallocCont(void *ptr, int contLen) { return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); } -void *rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg) { +void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg) { SRpcInfo *pRpc = (SRpcInfo *)shandle; SRpcReqContext *pContext; @@ -380,10 +380,12 @@ void *rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg) { || type == TSDB_MSG_TYPE_CM_SHOW ) pContext->connType = RPC_CONN_TCPC; + // set the handle to pContext, so app can cancel the request if (pMsg->handle) *((void **)pMsg->handle) = pContext; + rpcSendReqToServer(pRpc, pContext); - return pContext; + return; } void rpcSendResponse(const SRpcMsg *pRsp) { From 505b9a909161aef7f016f5def3a9e15700c75b5f Mon Sep 17 00:00:00 2001 From: dengyihao Date: Wed, 1 Jul 2020 02:38:17 +0800 Subject: [PATCH 03/18] fix test case --- tests/pytest/query/queryJoin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/pytest/query/queryJoin.py b/tests/pytest/query/queryJoin.py index bfc493580e..6d135e1006 100644 --- a/tests/pytest/query/queryJoin.py +++ b/tests/pytest/query/queryJoin.py @@ -112,8 +112,7 @@ class TDTestCase: tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id") tdSql.checkRows(6) - tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.pid, stb_p.id, stb_p.dscrption, stb_p.pressure,stb_v.velocity from stb_p, stb_t, stb_v where stb_p.ts=stb_t.ts and stb_p.ts=stb_v.ts and stb_p.id = stb_t.id") - tdSql.checkRows(2) + tdSql.error("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.pid, stb_p.id, stb_p.dscrption, stb_p.pressure,stb_v.velocity from stb_p, stb_t, stb_v where stb_p.ts=stb_t.ts and stb_p.ts=stb_v.ts and stb_p.id = stb_t.id") def stop(self): tdSql.close() From 37743a1fbf841b2732a9278bf684968eb7152b80 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 1 Jul 2020 10:08:11 +0800 Subject: [PATCH 04/18] [TD-811] --- src/kit/taosdemo/taosdemo.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 3e802c9566..3265285cca 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -808,27 +808,31 @@ void *readMetric(void *sarg) { } void queryDB(TAOS *taos, char *command) { - int i = 5; + int i; TAOS_RES *pSql = NULL; - int32_t code = -1; - while (i > 0 && code != 0) { + int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != pSql) { + taos_free_result(pSql); + pSql = NULL; + } + pSql = taos_query(taos, command); code = taos_errno(pSql); - taos_free_result(pSql); - pSql = NULL; - if (code == 0) { + if (0 == code) { break; - } - i--; + } } if (code != 0) { fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql)); taos_free_result(pSql); - taos_close(taos); exit(EXIT_FAILURE); } + + taos_free_result(pSql); } // sync insertion From c1252c2dbee0349290dda7e8aeaddaec1390d78a Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 1 Jul 2020 11:08:19 +0800 Subject: [PATCH 05/18] TD-803: use KV to transfer tags --- src/client/src/tscParseInsert.c | 43 +++++++++++++-------------------- src/client/src/tscSQLParser.c | 25 +++++++++++++------ src/tsdb/src/tsdbMeta.c | 22 +++-------------- 3 files changed, 38 insertions(+), 52 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index f0c8ffebc2..fd2a9f5a8c 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -891,11 +891,15 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return tscInvalidSQLErrMsg(pCmd->payload, "keyword TAGS expected", sToken.z); } + SKVRowBuilder kvRowBuilder = {0}; + if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + uint32_t ignoreTokenTypes = TK_LP; uint32_t numOfIgnoreToken = 1; for (int i = 0; i < spd.numOfAssignedCols; ++i) { - char * tagVal = pTag->data + spd.elems[i].offset; - int16_t colIndex = spd.elems[i].colIndex; + SSchema* pSchema = pTagSchema + spd.elems[i].colIndex; index = 0; sToken = tStrGetToken(sql, &index, true, numOfIgnoreToken, &ignoreTokenTypes); @@ -911,12 +915,21 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { sToken.n -= 2; } - code = tsParseOneColumnData(&pTagSchema[colIndex], &sToken, tagVal, pCmd->payload, &sql, false, tinfo.precision); + char tagVal[TSDB_MAX_TAGS_LEN]; + code = tsParseOneColumnData(pSchema, &sToken, tagVal, pCmd->payload, &sql, false, tinfo.precision); if (code != TSDB_CODE_SUCCESS) { return code; } + + tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); } + SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); + pTag->dataLen = kvRowLen(row); + memcpy(pTag->data, row, pTag->dataLen); + free(row); + tdDestroyKVRowBuilder(&kvRowBuilder); + index = 0; sToken = tStrGetToken(sql, &index, false, 0, NULL); sql += index; @@ -924,29 +937,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return tscInvalidSQLErrMsg(pCmd->payload, ") expected", sToken.z); } - // 2. set the null value for the columns that do not assign values - if (spd.numOfAssignedCols < spd.numOfCols) { - char *ptr = pTag->data; - - for (int32_t i = 0; i < spd.numOfCols; ++i) { - if (!spd.hasVal[i]) { // current tag column do not have any value to insert, set it to null - if (pTagSchema[i].type == TSDB_DATA_TYPE_BINARY || pTagSchema[i].type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(ptr, pTagSchema[i].type); - } else { - setNull(ptr, pTagSchema[i].type, pTagSchema[i].bytes); - } - } - - ptr += pTagSchema[i].bytes; - } - } - - // 3. calculate the actual data size of STagData - pCmd->payloadLen = sizeof(pTag->name) + sizeof(pTag->dataLen); - for (int32_t t = 0; t < numOfTags; ++t) { - pTag->dataLen += pTagSchema[t].bytes; - pCmd->payloadLen += pTagSchema[t].bytes; - } + pCmd->payloadLen = sizeof(pTag->name) + sizeof(pTag->dataLen) + pTag->dataLen; pTag->dataLen = htonl(pTag->dataLen); if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index df18d7a56b..25fb2968f7 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5623,25 +5623,36 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { SSchema* pTagSchema = tscGetTableTagSchema(pStableMeterMetaInfo->pTableMeta); STagData* pTag = &pCreateTable->usingInfo.tagdata; - char* tagVal = pTag->data; + SKVRowBuilder kvRowBuilder = {0}; + if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + int32_t ret = TSDB_CODE_SUCCESS; - for (int32_t i = 0; i < pList->nExpr; ++i) { - if (pTagSchema[i].type == TSDB_DATA_TYPE_BINARY || pTagSchema[i].type == TSDB_DATA_TYPE_NCHAR) { + SSchema* pSchema = pTagSchema + i; + if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { // validate the length of binary - if (pList->a[i].pVar.nLen + VARSTR_HEADER_SIZE > pTagSchema[i].bytes) { + if (pList->a[i].pVar.nLen + VARSTR_HEADER_SIZE > pSchema->bytes) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } - ret = tVariantDump(&(pList->a[i].pVar), tagVal, pTagSchema[i].type, true); + char tagVal[TSDB_MAX_TAGS_LEN]; + ret = tVariantDump(&(pList->a[i].pVar), tagVal, pSchema->type, true); if (ret != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } - tagVal += pTagSchema[i].bytes; + tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); } + SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); + pTag->dataLen = kvRowLen(row); + memcpy(pTag->data, row, pTag->dataLen); + free(row); + tdDestroyKVRowBuilder(&kvRowBuilder); + // table name if (tscValidateName(&pInfo->pCreateTableInfo->name) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); @@ -5653,7 +5664,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return ret; } - pTag->dataLen = tagVal - pTag->data; + //pTag->dataLen = tagVal - pTag->data; return TSDB_CODE_SUCCESS; } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 13b676ee1d..7851329c47 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -233,26 +233,10 @@ STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) { if (tsdbTableSetSName(pCfg, pMsg->superTableId, true) < 0) goto _err; if (tsdbTableSetSuperUid(pCfg, htobe64(pMsg->superTableUid)) < 0) goto _err; - // Decode tag values - if (pMsg->tagDataLen) { - int accBytes = 0; + int32_t tagDataLen = htonl(pMsg->tagDataLen); + if (tagDataLen) { char *pTagData = pMsg->data + (numOfCols + numOfTags) * sizeof(SSchema); - - SKVRowBuilder kvRowBuilder = {0}; - if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - for (int i = numOfCols; i < numOfCols + numOfTags; i++) { - if (tdAddColToKVRow(&kvRowBuilder, htons(pSchema[i].colId), pSchema[i].type, pTagData + accBytes) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - accBytes += htons(pSchema[i].bytes); - } - - tsdbTableSetTagValue(pCfg, tdGetKVRowFromBuilder(&kvRowBuilder), false); - tdDestroyKVRowBuilder(&kvRowBuilder); + tsdbTableSetTagValue(pCfg, pTagData, true); } } From 29adf9c752a0935a840d703549807a68805117d7 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 1 Jul 2020 11:17:48 +0800 Subject: [PATCH 06/18] TD-803: increase bufsize --- src/common/inc/tdataformat.h | 2 +- src/inc/taosmsg.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 1cd72eafde..da2f07da04 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -272,7 +272,7 @@ typedef struct { int16_t offset; } SColIdx; -#define TD_KV_ROW_HEAD_SIZE 2 * sizeof(int16_t) +#define TD_KV_ROW_HEAD_SIZE (2 * sizeof(int16_t)) #define kvRowLen(r) (*(int16_t *)(r)) #define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 13fa799b3f..cb25242d27 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -26,6 +26,7 @@ extern "C" { #include "taosdef.h" #include "taoserror.h" #include "trpc.h" +#include "tdataformat.h" // message type @@ -674,7 +675,7 @@ typedef struct SMultiTableMeta { typedef struct { int32_t dataLen; char name[TSDB_TABLE_ID_LEN]; - char data[TSDB_MAX_TAGS_LEN]; + char data[TSDB_MAX_TAGS_LEN + TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * TSDB_MAX_TAGS]; } STagData; /* From c14562506c6f72d84de94b4dc7bd37cc18eb9d85 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 1 Jul 2020 11:33:46 +0800 Subject: [PATCH 07/18] update perf-test script. --- tests/perftest-scripts/perftest-daily.sh | 127 +++++--- tests/perftest-scripts/perftest-taosdemo.sh | 73 +++-- .../perftest-tsdb-compare-13d.sh | 52 +++- .../perftest-tsdb-compare-1d.sh | 56 +++- .../perftest-tsdb-compare-var10k-int100s.sh | 84 ++++++ .../perftest-tsdb-compare-var10k-int10s.sh | 84 ++++++ tests/perftest-scripts/runInfluxdb.sh | 251 ++++++++++++++++ tests/perftest-scripts/runTDengine.sh | 283 ++++++++++++++++++ .../taosdemo-rps-csv2png.gnuplot | 24 ++ 9 files changed, 947 insertions(+), 87 deletions(-) create mode 100755 tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh create mode 100755 tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh create mode 100755 tests/perftest-scripts/runInfluxdb.sh create mode 100755 tests/perftest-scripts/runTDengine.sh create mode 100644 tests/perftest-scripts/taosdemo-rps-csv2png.gnuplot diff --git a/tests/perftest-scripts/perftest-daily.sh b/tests/perftest-scripts/perftest-daily.sh index b44387047c..beac8de15a 100755 --- a/tests/perftest-scripts/perftest-daily.sh +++ b/tests/perftest-scripts/perftest-daily.sh @@ -1,5 +1,7 @@ #!/bin/bash +WORK_DIR=/mnt/root + # Coloured Echoes # function red_echo { echo -e "\033[31m$@\033[0m"; } # function green_echo { echo -e "\033[32m$@\033[0m"; } # @@ -16,6 +18,17 @@ function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } # function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # +function setMaxTablesPerVnode { + echo "/etc/taos/taos.cfg maxTablesPerVnode will be set to $1" + + hasText=`grep "maxTablesPerVnode" /etc/taos/taos.cfg` + if [[ -z "$hasText" ]]; then + echo "maxTablesPerVnode $1" >> /etc/taos/taos.cfg + else + sed -i 's/^maxTablesPerVnode.*$/maxTablesPerVnode '"$1"'/g' /etc/taos/taos.cfg + fi +} + function setMaxConnections { echo "/etc/taos/taos.cfg maxConnection will be set to $1" @@ -27,6 +40,28 @@ function setMaxConnections { fi } +function setQDebugFlag { + echo "/etc/taos/taos.cfg qDebugFlag will be set to $1" + + hasText=`grep -w "qDebugFlag" /etc/taos/taos.cfg` + if [[ -z "$hasText" ]]; then + echo "qDebugFlag $1" >> /etc/taos/taos.cfg + else + sed -i 's/^qDebugFlag.*$/qDebugFlag '"$1"'/g' /etc/taos/taos.cfg + fi +} + +function setDebugFlag { + echo "/etc/taos/taos.cfg DebugFlag will be set to $1" + + hasText=`grep -w "DebugFlag" /etc/taos/taos.cfg` + if [[ -z "$hasText" ]]; then + echo "DebugFlag $1" >> /etc/taos/taos.cfg + else + sed -i 's/^DebugFlag.*$/DebugFlag '"$1"'/g' /etc/taos/taos.cfg + fi +} + function setWal { echo "/etc/taos/taos.cfg walLevel will be set to $1" @@ -47,9 +82,10 @@ function collectSysInfo { } function buildTDengine { - cd /root/TDengine + echoInfo "Build TDengine" + cd $WORK_DIR/TDengine - git remote update + git remote update > /dev/null REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop` LOCAL_COMMIT=`git rev-parse --short @` @@ -59,29 +95,17 @@ function buildTDengine { echo "repo up-to-date" else echo "repo need to pull" - git pull + git pull > /dev/null LOCAL_COMMIT=`git rev-parse --short @` cd debug rm -rf * - cmake .. + cmake .. > /dev/null make > /dev/null make install fi } -function restartTaosd { - systemctl stop taosd - pkill -KILL -x taosd - sleep 10 - - rm -rf /mnt/var/log/taos/* - rm -rf /mnt/var/lib/taos/* - - taosd 2>&1 > /dev/null & - sleep 10 -} - function sendReport { receiver="sdsang@taosdata.com, sangshuduo@gmail.com" mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" @@ -93,57 +117,90 @@ function sendReport { (cat - && uuencode perftest-13d-wal1-$today.log perftest-13d-wal1-$today.log)| \ (cat - && uuencode perftest-13d-wal1-report.csv perftest-13d-wal1-report-$today.csv) | \ (cat - && uuencode perftest-13d-wal1-report.png perftest-13d-wal1-report-$today.png) | \ + (cat - && uuencode perftest-var10k-int10s-wal1-$today.log perftest-var10k-int10s-wal1-$today.log)| \ + (cat - && uuencode perftest-var10k-int10s-wal1-report.csv perftest-var10k-int10s-wal1-report-$today.csv) | \ + (cat - && uuencode perftest-var10k-int10s-wal1-report.png perftest-var10k-int10s-wal1-report-$today.png) | \ (cat - && uuencode taosdemo-wal1-$today.log taosdemo-wal1-$today.log) | \ (cat - && uuencode taosdemo-wal1-report.csv taosdemo-wal1-report-$today.csv) | \ - (cat - && uuencode taosdemo-rps-wal1-report.csv taosdemo-rps-wal1-report-$today.csv) | \ (cat - && uuencode taosdemo-wal1-report.png taosdemo-wal1-report-$today.png) | \ + (cat - && uuencode taosdemo-rps-wal1-report.csv taosdemo-rps-wal1-report-$today.csv) | \ + (cat - && uuencode taosdemo-rps-wal1-report.png taosdemo-rps-wal1-report-$today.png) | \ (cat - && uuencode perftest-1d-wal2-$today.log perftest-1d-wal2-$today.log)| \ (cat - && uuencode perftest-1d-wal2-report.csv perftest-1d-wal2-report-$today.csv) | \ (cat - && uuencode perftest-1d-wal2-report.png perftest-1d-wal2-report-$today.png) | \ (cat - && uuencode perftest-13d-wal2-$today.log perftest-13d-wal2-$today.log)| \ (cat - && uuencode perftest-13d-wal2-report.csv perftest-13d-wal2-report-$today.csv) | \ (cat - && uuencode perftest-13d-wal2-report.png perftest-13d-wal2-report-$today.png) | \ + (cat - && uuencode perftest-var10k-int10s-wal2-$today.log perftest-var10k-int10s-wal2-$today.log)| \ + (cat - && uuencode perftest-var10k-int10s-wal2-report.csv perftest-var10k-int10s-wal2-report-$today.csv) | \ + (cat - && uuencode perftest-var10k-int10s-wal2-report.png perftest-var10k-int10s-wal2-report-$today.png) | \ (cat - && uuencode taosdemo-wal2-$today.log taosdemo-wal2-$today.log) | \ (cat - && uuencode taosdemo-wal2-report.csv taosdemo-wal2-report-$today.csv) | \ + (cat - && uuencode taosdemo-wal2-report.png taosdemo-wal2-report-$today.png) | \ (cat - && uuencode taosdemo-rps-wal2-report.csv taosdemo-rps-wal2-report-$today.csv) | \ + (cat - && uuencode taosdemo-rps-wal2-report.png taosdemo-rps-wal2-report-$today.png) | \ (cat - && uuencode sysinfo.log sysinfo.txt) | \ (cat - && uuencode taos.cfg taos-cfg-$today.txt) | \ ssmtp "${receiver}" } today=`date +"%Y%m%d"` -cd /root -echo -e "cron-ran-at-${today}" >> cron.log +cd $WORK_DIR +echo -e "cron-ran-at-${today}" >> $WORK_DIR/cron.log -echoInfo "Build TDengine" buildTDengine ############################ -setMaxConnections 100 +setMaxConnections 1000 +setMaxTablesPerVnode 6000 +setDebugFlag 131 +setQDebugFlag 131 ############################ setWal "2" -cd /root -./perftest-tsdb-compare-1d.sh "wal2" - -cd /root -./perftest-tsdb-compare-13d.sh "wal2" - -cd /root +cd $WORK_DIR +date >> $WORK_DIR/cron.log ./perftest-taosdemo.sh "wal2" +date >> $WORK_DIR/cron.log + +cd $WORK_DIR +date >> $WORK_DIR/cron.log +./perftest-tsdb-compare-1d.sh +date >> $WORK_DIR/cron.log + +cd $WORK_DIR +date >> $WORK_DIR/cron.log +./perftest-tsdb-compare-13d.sh +date >> $WORK_DIR/cron.log + +cd $WORK_DIR +date >> $WORK_DIR/cron.log +./perftest-tsdb-compare-var10k-int10s.sh +date >> $WORK_DIR/cron.log ############################# setWal "1" -cd /root -./perftest-tsdb-compare-1d.sh "wal1" - -cd /root -./perftest-tsdb-compare-13d.sh "wal1" - -cd /root +cd $WORK_DIR +date >> $WORK_DIR/cron.log ./perftest-taosdemo.sh "wal1" +date >> $WORK_DIR/cron.log + +cd $WORK_DIR +date >> $WORK_DIR/cron.log +./perftest-tsdb-compare-1d.sh +date >> $WORK_DIR/cron.log + +cd $WORK_DIR +date >> $WORK_DIR/cron.log +./perftest-tsdb-compare-13d.sh +date >> $WORK_DIR/cron.log + +cd $WORK_DIR +date >> $WORK_DIR/cron.log +./perftest-tsdb-compare-var10k-int10s.sh +date >> $WORK_DIR/cron.log ############################# collectSysInfo diff --git a/tests/perftest-scripts/perftest-taosdemo.sh b/tests/perftest-scripts/perftest-taosdemo.sh index 0dd9a0572a..e459769512 100755 --- a/tests/perftest-scripts/perftest-taosdemo.sh +++ b/tests/perftest-scripts/perftest-taosdemo.sh @@ -1,5 +1,20 @@ #!/bin/bash +WORK_DIR=/mnt/root + +walLevel=`grep "^walLevel" /etc/taos/taos.cfg | awk '{print $2}'` +if [[ "$walLevel" -eq "2" ]]; then + walPostfix="wal2" +elif [[ "$walLevel" -eq "1" ]]; then + walPostfix="wal1" +else + echo -e "${RED}wrong walLevel $walLevel found! ${NC}" + exit 1 +fi + +logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` +dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` + # Coloured Echoes function red_echo { echo -e "\033[31m$@\033[0m"; } function green_echo { echo -e "\033[32m$@\033[0m"; } @@ -17,13 +32,20 @@ function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } function restartTaosd { + echo "Stop taosd" systemctl stop taosd - pkill -KILL -x taosd - sleep 10 - - rm -rf /mnt/var/log/taos/* - rm -rf /mnt/var/lib/taos/* + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + + rm -rf $logDir/* + rm -rf $dataDir/* + echo "Start taosd" taosd 2>&1 > /dev/null & sleep 10 } @@ -32,7 +54,7 @@ function runCreateTableOnly { echoInfo "Restart Taosd" restartTaosd - /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -n 0 2>&1 | tee taosdemo-$1-$today.log" + /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -n 0 2>&1 | tee taosdemo-$walPostfix-$today.log" demoCreateTableOnly=`grep "Total:" totaltime.out|awk '{print $2}'` } @@ -40,7 +62,7 @@ function runDeleteTableOnly { echoInfo "Restart Taosd" restartTaosd - /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -t 0 -D 1 2>&1 | tee taosdemo-$1-$today.log" + /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -t 0 -D 1 2>&1 | tee taosdemo-$walPostfix-$today.log" demoDeleteTableOnly=`grep "Total:" totaltime.out|awk '{print $2}'` } @@ -48,41 +70,44 @@ function runCreateTableThenInsert { echoInfo "Restart Taosd" restartTaosd - /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo 2>&1 | tee -a taosdemo-$1-$today.log" + /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo 2>&1 | tee -a taosdemo-$walPostfix-$today.log" demoTableAndInsert=`grep "Total:" totaltime.out|awk '{print $2}'` - demoRPS=`grep "records\/second" taosdemo-$1-$today.log | tail -n1 | awk '{print $13}'` + demoRPS=`grep "records\/second" taosdemo-$walPostfix-$today.log | tail -n1 | awk '{print $13}'` } function generateTaosdemoPlot { - echo "${today} $1, demoCreateTableOnly: ${demoCreateTableOnly}, demoDeleteTableOnly: ${demoDeleteTableOnly}, demoTableAndInsert: ${demoTableAndInsert}" | tee -a taosdemo-$today.log - echo "${today}, ${demoCreateTableOnly}, ${demoDeleteTableOnly}, ${demoTableAndInsert}">> taosdemo-$1-report.csv - echo "${today}, ${demoRPS}" >> taosdemo-rps-$1-report.csv + echo "${today} $walPostfix, demoCreateTableOnly: ${demoCreateTableOnly}, demoDeleteTableOnly: ${demoDeleteTableOnly}, demoTableAndInsert: ${demoTableAndInsert}" | tee -a taosdemo-$today.log + echo "${today}, ${demoCreateTableOnly}, ${demoDeleteTableOnly}, ${demoTableAndInsert}">> taosdemo-$walPostfix-report.csv + echo "${today}, ${demoRPS}" >> taosdemo-rps-$walPostfix-report.csv - csvLines=`cat taosdemo-$1-report.csv | wc -l` + csvLines=`cat taosdemo-$walPostfix-report.csv | wc -l` if [ "$csvLines" -gt "10" ]; then - sed -i '1d' taosdemo-$1-report.csv + sed -i '1d' taosdemo-$walPostfix-report.csv fi - csvLines=`cat taosdemo-rps-$1-report.csv | wc -l` + csvLines=`cat taosdemo-rps-$walPostfix-report.csv | wc -l` if [ "$csvLines" -gt "10" ]; then - sed -i '1d' taosdemo-rps-$1-report.csv + sed -i '1d' taosdemo-rps-$walPostfix-report.csv fi - gnuplot -e "filename='taosdemo-$1-report'" -p taosdemo-csv2png.gnuplot - gnuplot -e "filename='taosdemo-rps-$1-report'" -p taosdemo-rps-csv2png.gnuplot + gnuplot -e "filename='taosdemo-$walPostfix-report'" -p taosdemo-csv2png.gnuplot + gnuplot -e "filename='taosdemo-rps-$walPostfix-report'" -p taosdemo-rps-csv2png.gnuplot } today=`date +"%Y%m%d"` -cd /root +cd $WORK_DIR echoInfo "Test Create Table Only " -runCreateTableOnly $1 +runCreateTableOnly echoInfo "Test Create Table then Insert data" -runDeleteTableOnly $1 +runDeleteTableOnly echoInfo "Test Create Table then Insert data" -runCreateTableThenInsert $1 +runCreateTableThenInsert echoInfo "Generate plot for taosdemo" -generateTaosdemoPlot $1 -echoInfo "End of TaosDemo Test" +generateTaosdemoPlot + +tar czf $WORK_DIR/taos-log-taosdemo-$today.tar.gz $logDir/* + +echoInfo "End of TaosDemo Test" | tee -a $WORK_DIR/cron.log diff --git a/tests/perftest-scripts/perftest-tsdb-compare-13d.sh b/tests/perftest-scripts/perftest-tsdb-compare-13d.sh index 8b326d2d41..110ab9e5fa 100755 --- a/tests/perftest-scripts/perftest-tsdb-compare-13d.sh +++ b/tests/perftest-scripts/perftest-tsdb-compare-13d.sh @@ -1,5 +1,21 @@ #!/bin/bash +WORK_DIR=/mnt/root +TSDB_CMP_DIR=timeseriesdatabase-comparisons/build/tsdbcompare + +walLevel=`grep "^walLevel" /etc/taos/taos.cfg | awk '{print $2}'` +if [[ "$walLevel" -eq "2" ]]; then + walPostfix="wal2" +elif [[ "$walLevel" -eq "1" ]]; then + walPostfix="wal1" +else + echo -e "${RED}wrong walLevel $walLevel found! ${NC}" + exit 1 +fi + +logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` +dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` + # Coloured Echoes # function red_echo { echo -e "\033[31m$@\033[0m"; } # function green_echo { echo -e "\033[32m$@\033[0m"; } # @@ -17,13 +33,20 @@ function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # function restartTaosd { + echo "Stop taosd" systemctl stop taosd - pkill -KILL -x taosd - sleep 10 - - rm -rf /mnt/var/log/taos/* - rm -rf /mnt/var/lib/taos/* + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + + rm -rf $logDir/* + rm -rf $dataDir/* + echo "Start taosd" taosd 2>&1 > /dev/null & sleep 10 } @@ -32,27 +55,30 @@ function runPerfTest13d { echoInfo "Restart Taosd" restartTaosd - cd /home/taos/tliu/timeseriesdatabase-comparisons/build/tsdbcompare - ./runreal-13d-csv.sh $1 2>&1 | tee /root/perftest-13d-$1-$today.log + cd $WORK_DIR/$TSDB_CMP_DIR + ./runTDengine.sh -d 13 -w -q 2>&1 | tee $WORK_DIR/perftest-13d-$walPostfix-$today.log } function generatePerfPlot13d { - cd /root + cd $WORK_DIR - csvLines=`cat perftest-13d-$1-report.csv | wc -l` + csvLines=`cat perftest-13d-$walPostfix-report.csv | wc -l` if [ "$csvLines" -gt "10" ]; then - sed -i '1d' perftest-13d-$1-report.csv + sed -i '1d' perftest-13d-$walPostfix-report.csv fi - gnuplot -e "filename='perftest-13d-$1-report'" -p perftest-csv2png.gnuplot + gnuplot -e "filename='perftest-13d-$walPostfix-report'" -p perftest-csv2png.gnuplot } today=`date +"%Y%m%d"` -cd /root +cd $WORK_DIR echoInfo "run Performance Test with 13 days data" runPerfTest13d $1 echoInfo "Generate plot of 13 days data" generatePerfPlot13d $1 -echoInfo "End of TSDB-Compare 13-days-data Test" + +tar czf $WORK_DIR/taos-log-13d-$today.tar.gz $logDir/* + +echoInfo "End of TSDB-Compare 13-days-data Test" | tee -a $WORK_DIR/cron.log diff --git a/tests/perftest-scripts/perftest-tsdb-compare-1d.sh b/tests/perftest-scripts/perftest-tsdb-compare-1d.sh index 0931480e21..9e8bc697bc 100755 --- a/tests/perftest-scripts/perftest-tsdb-compare-1d.sh +++ b/tests/perftest-scripts/perftest-tsdb-compare-1d.sh @@ -1,5 +1,21 @@ #!/bin/bash +WORK_DIR=/mnt/root +TSDB_CMP_DIR=timeseriesdatabase-comparisons/build/tsdbcompare + +walLevel=`grep "^walLevel" /etc/taos/taos.cfg | awk '{print $2}'` +if [[ "$walLevel" -eq "2" ]]; then + walPostfix="wal2" +elif [[ "$walLevel" -eq "1" ]]; then + walPostfix="wal1" +else + echo -e "${RED}wrong walLevel $walLevel found! ${NC}" + exit 1 +fi + +logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` +dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` + # Coloured Echoes # function red_echo { echo -e "\033[31m$@\033[0m"; } # function green_echo { echo -e "\033[32m$@\033[0m"; } # @@ -17,13 +33,20 @@ function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # function restartTaosd { + echo "Stop taosd" systemctl stop taosd - pkill -KILL -x taosd - sleep 10 - - rm -rf /mnt/var/log/taos/* - rm -rf /mnt/var/lib/taos/* + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + + rm -rf $logDir/* + rm -rf $dataDir/* + echo "Start taosd" taosd 2>&1 > /dev/null & sleep 10 } @@ -32,27 +55,30 @@ function runPerfTest1d { echoInfo "Restart Taosd" restartTaosd - cd /home/taos/tliu/timeseriesdatabase-comparisons/build/tsdbcompare - ./runreal-1d-csv.sh $1 2>&1 | tee /root/perftest-1d-$1-$today.log + cd $WORK_DIR/$TSDB_CMP_DIR + ./runTDengine.sh -d 1 -w -q 2>&1 | tee $WORK_DIR/perftest-1d-$walPostfix-$today.log } function generatePerfPlot1d { - cd /root + cd $WORK_DIR - csvLines=`cat perftest-1d-$1-report.csv | wc -l` + csvLines=`cat perftest-1d-$walPostfix-report.csv | wc -l` if [ "$csvLines" -gt "10" ]; then - sed -i '2d' perftest-1d-$1-report.csv + sed -i '1d' perftest-1d-$walPostfix-report.csv fi - gnuplot -e "filename='perftest-1d-$1-report'" -p perftest-csv2png.gnuplot + gnuplot -e "filename='perftest-1d-$walPostfix-report'" -p perftest-csv2png.gnuplot } today=`date +"%Y%m%d"` -cd /root +cd $WORK_DIR echoInfo "run Performance Test with 1 day data" -runPerfTest1d $1 +runPerfTest1d echoInfo "Generate plot of 1 day data" -generatePerfPlot1d $1 -echoInfo "End of TSDB-Compare 1-day-data Test" +generatePerfPlot1d + +tar czf $WORK_DIR/taos-log-1d-$today.tar.gz $logDir/* + +echoInfo "End of TSDB-Compare 1-day-data Test" | tee -a $WORK_DIR/cron.log diff --git a/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh b/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh new file mode 100755 index 0000000000..5fc5b9d03a --- /dev/null +++ b/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +WORK_DIR=/mnt/root +TSDB_CMP_DIR=timeseriesdatabase-comparisons/build/tsdbcompare + +walLevel=`grep "^walLevel" /etc/taos/taos.cfg | awk '{print $2}'` +if [[ "$walLevel" -eq "2" ]]; then + walPostfix="wal2" +elif [[ "$walLevel" -eq "1" ]]; then + walPostfix="wal1" +else + echo -e "${RED}wrong walLevel $walLevel found! ${NC}" + exit 1 +fi + +logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` +dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` + +# Coloured Echoes # +function red_echo { echo -e "\033[31m$@\033[0m"; } # +function green_echo { echo -e "\033[32m$@\033[0m"; } # +function yellow_echo { echo -e "\033[33m$@\033[0m"; } # +function white_echo { echo -e "\033[1;37m$@\033[0m"; } # +# Coloured Printfs # +function red_printf { printf "\033[31m$@\033[0m"; } # +function green_printf { printf "\033[32m$@\033[0m"; } # +function yellow_printf { printf "\033[33m$@\033[0m"; } # +function white_printf { printf "\033[1;37m$@\033[0m"; } # +# Debugging Outputs # +function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; } # +function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; } # +function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } # +function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # + +function restartTaosd { + echo "Stop taosd" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + + rm -rf $logDir/* + rm -rf $dataDir/* + + echo "Start taosd" + taosd 2>&1 > /dev/null & + sleep 10 +} + +function runPerfTestVar10K { + echoInfo "Restart Taosd" + restartTaosd + + cd $WORK_DIR/$TSDB_CMP_DIR + ./runTDengine.sh -v 10000 -i 100 -w -q 2>&1 | tee $WORK_DIR/perftest-var10k-int100s-$walPostfix-$today.log +} + +function generatePerfPlotVar10K { + cd $WORK_DIR + + csvLines=`cat perftest-var10k-int100s-$walPostfix-report.csv | wc -l` + + if [ "$csvLines" -gt "10" ]; then + sed -i '1d' perftest-var10k-int100s-$walPostfix-report.csv + fi + + gnuplot -e "filename='perftest-var10k-int100s-$walPostfix-report'" -p perftest-csv2png.gnuplot +} + +today=`date +"%Y%m%d"` +cd $WORK_DIR + +echoInfo "run Performance Test with 10K tables data" +runPerfTestVar10K +echoInfo "Generate plot of 10K tables data" +generatePerfPlotVar10K + +tar czf $WORK_DIR/taos-log-var10k-int100s-$today.tar.gz $logDir/* + +echoInfo "End of TSDB-Compare var10k-int100s-tables-data Test" | tee -a $WORK_DIR/cron.log diff --git a/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh b/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh new file mode 100755 index 0000000000..bafa04f174 --- /dev/null +++ b/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +WORK_DIR=/mnt/root +TSDB_CMP_DIR=timeseriesdatabase-comparisons/build/tsdbcompare + +walLevel=`grep "^walLevel" /etc/taos/taos.cfg | awk '{print $2}'` +if [[ "$walLevel" -eq "2" ]]; then + walPostfix="wal2" +elif [[ "$walLevel" -eq "1" ]]; then + walPostfix="wal1" +else + echo -e "${RED}wrong walLevel $walLevel found! ${NC}" + exit 1 +fi + +logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` +dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` + +# Coloured Echoes # +function red_echo { echo -e "\033[31m$@\033[0m"; } # +function green_echo { echo -e "\033[32m$@\033[0m"; } # +function yellow_echo { echo -e "\033[33m$@\033[0m"; } # +function white_echo { echo -e "\033[1;37m$@\033[0m"; } # +# Coloured Printfs # +function red_printf { printf "\033[31m$@\033[0m"; } # +function green_printf { printf "\033[32m$@\033[0m"; } # +function yellow_printf { printf "\033[33m$@\033[0m"; } # +function white_printf { printf "\033[1;37m$@\033[0m"; } # +# Debugging Outputs # +function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; } # +function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; } # +function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } # +function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # + +function restartTaosd { + echo "Stop taosd" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + + rm -rf $logDir/* + rm -rf $dataDir/* + + echo "Start taosd" + taosd 2>&1 > /dev/null & + sleep 10 +} + +function runPerfTestVar10K { + echoInfo "Restart Taosd" + restartTaosd + + cd $WORK_DIR/$TSDB_CMP_DIR + ./runTDengine.sh -v 10000 -w -q 2>&1 | tee $WORK_DIR/perftest-var10k-int10s-$walPostfix-$today.log +} + +function generatePerfPlotVar10K { + cd $WORK_DIR + + csvLines=`cat perftest-var10k-int10s-$walPostfix-report.csv | wc -l` + + if [ "$csvLines" -gt "10" ]; then + sed -i '1d' perftest-var10k-int10s-$walPostfix-report.csv + fi + + gnuplot -e "filename='perftest-var10k-int10s-$walPostfix-report'" -p perftest-csv2png.gnuplot +} + +today=`date +"%Y%m%d"` +cd $WORK_DIR + +echoInfo "run Performance Test with 10K tables data" +runPerfTestVar10K +echoInfo "Generate plot of 10K tables data" +generatePerfPlotVar10K + +tar czf $WORK_DIR/taos-log-var10k-int10s-$today.tar.gz $logDir/* + +echoInfo "End of TSDB-Compare var10k-int10s-tables-data Test" | tee -a $WORK_DIR/cron.log diff --git a/tests/perftest-scripts/runInfluxdb.sh b/tests/perftest-scripts/runInfluxdb.sh new file mode 100755 index 0000000000..e8975b05cf --- /dev/null +++ b/tests/perftest-scripts/runInfluxdb.sh @@ -0,0 +1,251 @@ +#!/bin/bash +#set -x + +WORK_DIR=/mnt/root +DATA_DIR=/mnt/data + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +# default value +DEFAULT_BATCH=5000 +DEFAULT_DAYS=1 +DEFAULT_INTERVAL=1 +DEFAULT_SCALEVAR=10 +DEFAULT_DOPREPARE=false +DEFAULT_DOWRITE=false +DEFAULT_DOQUERY=false + +# function +function do_prepare { + echo + echo "---------------Generating Data-----------------" + echo + + echo + echo "Prepare data for InfluxDB...." + + echo "bin/bulk_data_gen -seed 123 -format influx-bulk -sampling-interval $interval_s \ + -scale-var $scalevar -use-case devops -timestamp-start $TIME_START \ + -timestamp-end $TIME_END > $DATA_FILE" + + bin/bulk_data_gen -seed 123 -format influx-bulk -sampling-interval $interval_s \ + -scale-var $scalevar -use-case devops -timestamp-start $TIME_START \ + -timestamp-end $TIME_END > $DATA_FILE +} + +function do_write { + echo "cat $DATA_FILE | bin/bulk_load_influx \ + --batch-size=$batch --workers=20 --urls=http://172.15.1.5:8086 | grep loaded" + INFLUXRES=`cat $DATA_FILE | bin/bulk_load_influx \ + --batch-size=$batch --workers=20 --urls="http://172.15.1.5:8086" | grep loaded` + + echo -e "${GREEN}InfluxDB writing result:${NC}" + echo -e "${GREEN}$INFLUXRES${NC}" + DATA=`echo $INFLUXRES|awk '{print($2)}'` + TMP=`echo $INFLUXRES|awk '{print($5)}'` + IFWTM=`echo ${TMP%s*}` +} + +function do_query { + + echo + echo "------------------Querying Data-----------------" + echo + + echo + echo "start query test, query max from 8 hosts group by 1 hour, InfluxDB" + echo + +#Test case 1 +#测试用例1，查询所有数据中，用8个hostname标签进行匹配，匹配出这8个hostname对应的模拟服务器CPU数据中的usage_user这个监控数据的最大值。 +#select max(usage_user) from cpu where(hostname='host_a' and hostname='host_b'and hostname='host_c'and hostname='host_d'and hostname='host_e'and hostname='host_f' and hostname='host_g'and hostname='host_h') ; +# a,b,c,d,e,f,g,h are random 8 numbers. + echo "IFQS1=bin/bulk_query_gen -seed 123 -format influx-http -query-type 8-host-all -scale-var $scalevar -queries 1000 | bin/query_benchmarker_influxdb -urls="http://172.15.1.5:8086" -workers 50 -print-interval 0|grep wall" + + IFQS1=`bin/bulk_query_gen -seed 123 -format influx-http -query-type 8-host-all -scale-var $scalevar -queries 1000 | bin/query_benchmarker_influxdb -urls="http://172.15.1.5:8086" -workers 50 -print-interval 0|grep wall` + echo -e "${GREEN}InfluxDB query test case 1 result:${NC}" + echo -e "${GREEN}$IFQS1${NC}" + TMP=`echo $IFQS1|awk '{print($4)}'` + IFQ1=`echo ${TMP%s*}` + +#Test case 2 +#测试用例2，查询所有数据中，用8个hostname标签进行匹配，匹配出这8个hostname对应的模拟服务器CPU数据中的usage_user这个监控数据，以1小时为粒度，查询每1小时的最大值。 +#select max(usage_user) from cpu where(hostname='host_a' and hostname='host_b'and hostname='host_c'and hostname='host_d'and hostname='host_e'and hostname='host_f' and hostname='host_g'and hostname='host_h') interval(1h); +# a,b,c,d,e,f,g,h are random 8 numbers + echo "IFQS2=bin/bulk_query_gen -seed 123 -format influx-http -query-type 8-host-allbyhr -scale-var $scalevar -queries 1000 | bin/query_benchmarker_influxdb -urls=http://172.15.1.5:8086 -workers 50 -print-interval 0|grep wall" + + IFQS2=`bin/bulk_query_gen -seed 123 -format influx-http -query-type 8-host-allbyhr \ + -scale-var $scalevar -queries 1000 | bin/query_benchmarker_influxdb \ + -urls="http://172.15.1.5:8086" -workers 50 -print-interval 0|grep wall` + echo -e "${GREEN}InfluxDB query test case 2 result:${NC}" + echo -e "${GREEN}$IFQS2${NC}" + TMP=`echo $IFQS2|awk '{print($4)}'` + IFQ2=`echo ${TMP%s*}` + +#Test case 3 +#测试用例3，测试用例3，随机查询12个小时的数据，用8个hostname标签进行匹配，匹配出这8个hostname对应的模拟服务器CPU数据中的usage_user这个监控数据，以10分钟为粒度，查询每10分钟的最大值 +#select max(usage_user) from cpu where(hostname='host_a' and hostname='host_b'and hostname='host_c'and hostname='host_d'and hostname='host_e'and hostname='host_f' and hostname='host_g'and hostname='host_h') and time >x and time x and time >/dev/null 2>&1 +INFLUX=`docker run -d -p 8086:8086 --net tsdbcomp --ip 172.15.1.5 influxdb` >>/dev/null 2>&1 +sleep 10 + +if $dowrite; +then + echo -e "Start test InfluxDB writting, result in ${GREEN}Green line${NC}" + do_write +fi + +if $doquery; +then + echo -e "Start test InfluxDB query, result in ${GREEN}Green line${NC}" + do_query +fi + +echo +echo +echo "======================================================" +echo " tsdb performance comparision " +echo "======================================================" +if $dowrite; +then + echo -e " Writing $DATA records test takes: " + printf " InfluxDB | %-4.5f Seconds \n" $IFWTM + echo "------------------------------------------------------" +fi + +if $doquery; +then + echo " Query test cases: " + echo " case 1: select the max(value) from all data " + echo " filtered out 8 hosts " + echo " Query test case 1 takes: " + printf " InfluxDB | %-4.5f Seconds \n" $IFQ1 + echo "------------------------------------------------------" + echo " case 2: select the max(value) from all data " + echo " filtered out 8 hosts with an interval of 1 hour " + echo " case 2 takes: " + printf " InfluxDB | %-4.5f Seconds \n" $IFQ2 + echo "------------------------------------------------------" + echo " case 3: select the max(value) from random 12 hours" + echo " data filtered out 8 hosts with an interval of 10 min " + echo " filtered out 8 hosts interval(1h) " + echo " case 3 takes: " + printf " InfluxDB | %-4.5f Seconds \n" $IFQ3 + echo "------------------------------------------------------" + echo " case 4: select the max(value) from random 1 hour data " + echo " data filtered out 8 hosts with an interval of 1 min " + echo " case 4 takes: " + printf " InfluxDB | %-4.5f Seconds \n" $IFQ4 + echo "------------------------------------------------------" +fi + +docker stop $INFLUX >>/dev/null 2>&1 +docker container rm -f $INFLUX >>/dev/null 2>&1 +docker network rm tsdbcomp >>/dev/null 2>&1 + +today=`date +"%Y%m%d"` +echo "${today}, ${IFWTM}, ${IFQ1}, ${IFQ2}, ${IFQ3}, ${IFQ4}" >> $RECORD_CSV_FILE diff --git a/tests/perftest-scripts/runTDengine.sh b/tests/perftest-scripts/runTDengine.sh new file mode 100755 index 0000000000..6c5cf4e55e --- /dev/null +++ b/tests/perftest-scripts/runTDengine.sh @@ -0,0 +1,283 @@ +#!/bin/bash +#set -x + +WORK_DIR=/mnt/root +DATA_DIR=/mnt/data + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +# default value +DEFAULT_BATCH=5000 +DEFAULT_DAYS=1 +DEFAULT_INTERVAL=1 +DEFAULT_SCALEVAR=10 +DEFAULT_DOPREPARE=false +DEFAULT_DOWRITE=false +DEFAULT_DOQUERY=false + +# function +function do_prepare { + echo + echo "---------------Generating Data-----------------" + echo + + echo + echo "Prepare data for TDengine...." + + # bin/bulk_data_gen -seed 123 -format tdengine -tdschema-file config/TDengineSchema.toml -scale-var 100 -use-case devops -timestamp-start "2018-01-01T00:00:00Z" -timestamp-end "2018-01-02T00:00:00Z" > $DATA_DIR/tdengine.dat + echo "bin/bulk_data_gen -seed 123 -format tdengine -sampling-interval $interval_s \ + -tdschema-file config/TDengineSchema.toml -scale-var $scalevar \ + -use-case devops -timestamp-start $TIME_START \ + -timestamp-end $TIME_END \ + > $DATA_FILE" + + bin/bulk_data_gen -seed 123 -format tdengine -sampling-interval $interval_s \ + -tdschema-file config/TDengineSchema.toml -scale-var $scalevar \ + -use-case devops -timestamp-start $TIME_START \ + -timestamp-end $TIME_END \ + > $DATA_FILE +} + +function do_write { + echo "TDENGINERES=cat $DATA_FILE |bin/bulk_load_tdengine --url 127.0.0.1:0 \ + --batch-size $batch -do-load -report-tags n1 -workers 20 -fileout=false| grep loaded" + + TDENGINERES=`cat $DATA_FILE |bin/bulk_load_tdengine --url 127.0.0.1:0 \ + --batch-size $batch -do-load -report-tags n1 -workers 20 -fileout=false| grep loaded` + + echo + echo -e "${GREEN}TDengine writing result:${NC}" + echo -e "${GREEN}$TDENGINERES${NC}" + DATA=`echo $TDENGINERES|awk '{print($2)}'` + TMP=`echo $TDENGINERES|awk '{print($5)}'` + TDWTM=`echo ${TMP%s*}` + +} + +function do_query { + + echo + echo "------------------Querying Data-----------------" + echo + + echo + echo "start query test, query max from 8 hosts group by 1 hour, TDengine" + echo + +#Test case 1 +#测试用例1，查询所有数据中，用8个hostname标签进行匹配，匹配出这8个hostname对应的模拟服务器CPU数据中的usage_user这个监控数据的最大值。 +#select max(usage_user) from cpu where(hostname='host_a' and hostname='host_b'and hostname='host_c'and hostname='host_d'and hostname='host_e'and hostname='host_f' and hostname='host_g'and hostname='host_h') ; +# a,b,c,d,e,f,g,h are random 8 numbers. + echo "TDQS1=bin/bulk_query_gen -seed 123 -format tdengine -query-type 8-host-all \ + -scale-var $scalevar -queries 1000 | bin/query_benchmarker_tdengine \ + -urls=http://127.0.0.1:6020 -workers 50 -print-interval 0|grep wall" + + TDQS1=`bin/bulk_query_gen -seed 123 -format tdengine -query-type 8-host-all \ + -scale-var $scalevar -queries 1000 | bin/query_benchmarker_tdengine \ + -urls="http://127.0.0.1:6020" -workers 50 -print-interval 0|grep wall` + echo + echo -e "${GREEN}TDengine query test case 1 result:${NC}" + echo -e "${GREEN}$TDQS1${NC}" + TMP=`echo $TDQS1|awk '{print($4)}'` + TDQ1=`echo ${TMP%s*}` + +#Test case 2 +#测试用例2，查询所有数据中，用8个hostname标签进行匹配，匹配出这8个hostname对应的模拟服务器CPU数据中的usage_user这个监控数据，以1小时为粒度，查询每1小时的最大值。 +#select max(usage_user) from cpu where(hostname='host_a' and hostname='host_b'and hostname='host_c'and hostname='host_d'and hostname='host_e'and hostname='host_f' and hostname='host_g'and hostname='host_h') interval(1h); +# a,b,c,d,e,f,g,h are random 8 numbers + echo "TDQS2=bin/bulk_query_gen -seed 123 -format tdengine -query-type 8-host-allbyhr \ + -scale-var $scalevar -queries 1000 | bin/query_benchmarker_tdengine \ + -urls=http://127.0.0.1:6020 -workers 50 -print-interval 0|grep wall" + + TDQS2=`bin/bulk_query_gen -seed 123 -format tdengine -query-type 8-host-allbyhr \ + -scale-var $scalevar -queries 1000 | bin/query_benchmarker_tdengine \ + -urls="http://127.0.0.1:6020" -workers 50 -print-interval 0|grep wall` + + echo + echo -e "${GREEN}TDengine query test case 2 result:${NC}" + echo -e "${GREEN}$TDQS2${NC}" + TMP=`echo $TDQS2|awk '{print($4)}'` + TDQ2=`echo ${TMP%s*}` + +#Test case 3 +#测试用例3，测试用例3，随机查询12个小时的数据，用8个hostname标签进行匹配，匹配出这8个hostname对应的模拟服务器CPU数据中的usage_user这个监控数据，以10分钟为粒度，查询每10分钟的最大值 +#select max(usage_user) from cpu where(hostname='host_a' and hostname='host_b'and hostname='host_c'and hostname='host_d'and hostname='host_e'and hostname='host_f' and hostname='host_g'and hostname='host_h') and time >x and time x and time > $RECORD_CSV_FILE diff --git a/tests/perftest-scripts/taosdemo-rps-csv2png.gnuplot b/tests/perftest-scripts/taosdemo-rps-csv2png.gnuplot new file mode 100644 index 0000000000..a7fcb4bd43 --- /dev/null +++ b/tests/perftest-scripts/taosdemo-rps-csv2png.gnuplot @@ -0,0 +1,24 @@ +#!/user/bin/gnuplot +reset +set terminal png + +set title filename font ",20" + +set ylabel "Time in Seconds" + +set xdata time +set timefmt "%Y%m%d" +set format x "%Y-%m-%d" +set xlabel "Date" + +set style data linespoints + +set terminal pngcairo size 1024,768 enhanced font 'Segoe UI, 10' +set output filename . '.png' +set datafile separator ',' + +set key reverse Left outside +set grid + + +plot filename . '.csv' using 1:2 title "Request Per Second" From 13f8b063e5090ebad8547b3defbcb9cfb560ef93 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 1 Jul 2020 11:49:41 +0800 Subject: [PATCH 08/18] fix crash when there are 2 or more binary columns --- src/client/src/tscUtil.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 259bcd4cbd..26a81c597f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -625,18 +625,31 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo return len; } +static int32_t getRowExpandSize(STableMeta* pTableMeta) { + int32_t result = TD_DATA_ROW_HEAD_SIZE; + int32_t columns = tscGetNumOfColumns(pTableMeta); + SSchema* pSchema = tscGetTableSchema(pTableMeta); + for(int32_t i = 0; i < columns; i++) { + if (IS_VAR_DATA_TYPE((pSchema + i)->type)) { + result += TYPE_BYTES[TSDB_DATA_TYPE_BINARY]; + } + } + return result; +} + int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { SSqlCmd* pCmd = &pSql->cmd; // the maximum expanded size in byte when a row-wise data is converted to SDataRow format - const int32_t MAX_EXPAND_SIZE = TD_DATA_ROW_HEAD_SIZE + TYPE_BYTES[TSDB_DATA_TYPE_BINARY]; + STableDataBlocks* pOneTableBlock = taosArrayGetP(pTableDataBlockList, 0); + int32_t expandSize = getRowExpandSize(pOneTableBlock->pTableMeta); void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); size_t total = taosArrayGetSize(pTableDataBlockList); for (int32_t i = 0; i < total; ++i) { - STableDataBlocks* pOneTableBlock = taosArrayGetP(pTableDataBlockList, i); + pOneTableBlock = taosArrayGetP(pTableDataBlockList, i); STableDataBlocks* dataBuf = NULL; int32_t ret = @@ -650,7 +663,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { } SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; - int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * MAX_EXPAND_SIZE; + int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize; if (dataBuf->nAllocSize < destSize) { while (dataBuf->nAllocSize < destSize) { @@ -678,8 +691,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { tscDebug("%p tableId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableId, pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey)); - - int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + MAX_EXPAND_SIZE); + int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + expandSize); pBlocks->tid = htonl(pBlocks->tid); pBlocks->uid = htobe64(pBlocks->uid); From c43321fceafcf0a358fca0af9506e2a3a67f0e63 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 1 Jul 2020 11:53:48 +0800 Subject: [PATCH 09/18] fix a runtime error --- tests/pytest/query/queryMetaData.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/query/queryMetaData.py b/tests/pytest/query/queryMetaData.py index 01c63c3f4a..8fb9d9bf3a 100755 --- a/tests/pytest/query/queryMetaData.py +++ b/tests/pytest/query/queryMetaData.py @@ -58,7 +58,7 @@ class MetadataQuery: cursor.execute("use test") base = threadID * self.tables - tablesPerThread = (int) self.tables / self.numOfTherads + tablesPerThread = int (self.tables / self.numOfTherads) for i in range(tablesPerThread): cursor.execute( '''create table t%d using meters tags( From e0acd602e0c4f3af87ba34cbad85ca85594705f5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 1 Jul 2020 13:31:26 +0800 Subject: [PATCH 10/18] [TD-814] invalid read while close http connect --- src/plugins/http/src/httpContext.c | 9 +++++++-- src/plugins/http/src/httpHandle.c | 24 +++++++++++------------ src/plugins/http/src/httpJson.c | 16 +++++++-------- src/plugins/http/src/httpServer.c | 2 -- src/plugins/http/src/httpSystem.c | 6 ++++-- src/util/src/tcache.c | 31 ++++++++++++++++-------------- 6 files changed, 48 insertions(+), 40 deletions(-) diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index b078be5930..b09f34b562 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -141,10 +141,15 @@ HttpContext *httpGetContext(void *ptr) { void httpReleaseContext(HttpContext *pContext) { int32_t refCount = atomic_sub_fetch_32(&pContext->refCount, 1); assert(refCount >= 0); - httpDebug("context:%p, fd:%d, is releasd, refCount:%d", pContext, pContext->fd, refCount); + httpDebug("context:%p, is releasd, refCount:%d", pContext, refCount); HttpContext **ppContext = pContext->ppContext; - taosCacheRelease(tsHttpServer.contextCache, (void **)(&ppContext), false); + if (tsHttpServer.contextCache != NULL) { + taosCacheRelease(tsHttpServer.contextCache, (void **)(&ppContext), false); + } else { + httpDebug("context:%p, won't be destroyed for cache is already released", pContext); + // httpDestroyContext((void **)(&ppContext)); + } } bool httpInitContext(HttpContext *pContext) { diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c index 758286344a..056fe425d4 100644 --- a/src/plugins/http/src/httpHandle.c +++ b/src/plugins/http/src/httpHandle.c @@ -157,7 +157,7 @@ bool httpGetHttpMethod(HttpContext* pContext) { pParser->method.pos[pParser->method.len] = 0; pParser->pLast = pSeek + 1; - httpDebug("context:%p, fd:%d, ip:%s, httpMethod:%s", pContext, pContext->fd, pContext->ipstr, pParser->method.pos); + httpTrace("context:%p, fd:%d, ip:%s, httpMethod:%s", pContext, pContext->fd, pContext->ipstr, pParser->method.pos); return true; } @@ -186,23 +186,23 @@ bool httpParseHead(HttpContext* pContext) { HttpParser* pParser = &pContext->parser; if (strncasecmp(pParser->pLast, "Content-Length: ", 16) == 0) { pParser->data.len = (int32_t)atoi(pParser->pLast + 16); - httpDebug("context:%p, fd:%d, ip:%s, Content-Length:%d", pContext, pContext->fd, pContext->ipstr, + httpTrace("context:%p, fd:%d, ip:%s, Content-Length:%d", pContext, pContext->fd, pContext->ipstr, pParser->data.len); } else if (strncasecmp(pParser->pLast, "Accept-Encoding: ", 17) == 0) { if (tsHttpEnableCompress && strstr(pParser->pLast + 17, "gzip") != NULL) { pContext->acceptEncoding = HTTP_COMPRESS_GZIP; - httpDebug("context:%p, fd:%d, ip:%s, Accept-Encoding:gzip", pContext, pContext->fd, pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, Accept-Encoding:gzip", pContext, pContext->fd, pContext->ipstr); } else { pContext->acceptEncoding = HTTP_COMPRESS_IDENTITY; - httpDebug("context:%p, fd:%d, ip:%s, Accept-Encoding:identity", pContext, pContext->fd, pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, Accept-Encoding:identity", pContext, pContext->fd, pContext->ipstr); } } else if (strncasecmp(pParser->pLast, "Content-Encoding: ", 18) == 0) { if (strstr(pParser->pLast + 18, "gzip") != NULL) { pContext->contentEncoding = HTTP_COMPRESS_GZIP; - httpDebug("context:%p, fd:%d, ip:%s, Content-Encoding:gzip", pContext, pContext->fd, pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, Content-Encoding:gzip", pContext, pContext->fd, pContext->ipstr); } else { pContext->contentEncoding = HTTP_COMPRESS_IDENTITY; - httpDebug("context:%p, fd:%d, ip:%s, Content-Encoding:identity", pContext, pContext->fd, pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, Content-Encoding:identity", pContext, pContext->fd, pContext->ipstr); } } else if (strncasecmp(pParser->pLast, "Connection: ", 12) == 0) { if (strncasecmp(pParser->pLast + 12, "Keep-Alive", 10) == 0) { @@ -210,7 +210,7 @@ bool httpParseHead(HttpContext* pContext) { } else { pContext->httpKeepAlive = HTTP_KEEPALIVE_DISABLE; } - httpDebug("context:%p, fd:%d, ip:%s, keepAlive:%d", pContext, pContext->fd, pContext->ipstr, + httpTrace("context:%p, fd:%d, ip:%s, keepAlive:%d", pContext, pContext->fd, pContext->ipstr, pContext->httpKeepAlive); } else if (strncasecmp(pParser->pLast, "Transfer-Encoding: ", 19) == 0) { if (strncasecmp(pParser->pLast + 19, "chunked", 7) == 0) { @@ -281,7 +281,7 @@ bool httpReadChunkedBody(HttpContext* pContext, HttpParser* pParser) { httpParseChunkedBody(pContext, pParser, false); return HTTP_CHECK_BODY_SUCCESS; } else { - httpDebug("context:%p, fd:%d, ip:%s, chunked body not finished, continue read", pContext, pContext->fd, pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, chunked body not finished, continue read", pContext, pContext->fd, pContext->ipstr); if (!httpReadDataImp(pContext)) { httpError("context:%p, fd:%d, ip:%s, read chunked request error", pContext, pContext->fd, pContext->ipstr); return HTTP_CHECK_BODY_ERROR; @@ -299,7 +299,7 @@ int httpReadUnChunkedBody(HttpContext* pContext, HttpParser* pParser) { httpSendErrorResp(pContext, HTTP_PARSE_BODY_ERROR); return HTTP_CHECK_BODY_ERROR; } else if (dataReadLen < pParser->data.len) { - httpDebug("context:%p, fd:%d, ip:%s, un-chunked body not finished, read size:%d dataReadLen:%d < pContext->data.len:%d, continue read", + httpTrace("context:%p, fd:%d, ip:%s, un-chunked body not finished, read size:%d dataReadLen:%d < pContext->data.len:%d, continue read", pContext, pContext->fd, pContext->ipstr, pContext->parser.bufsize, dataReadLen, pParser->data.len); return HTTP_CHECK_BODY_CONTINUE; } else { @@ -313,9 +313,9 @@ bool httpParseRequest(HttpContext* pContext) { return true; } - httpDebug("context:%p, fd:%d, ip:%s, thread:%s, numOfFds:%d, read size:%d, raw data:\n%s", - pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->pThread->numOfFds, - pContext->parser.bufsize, pContext->parser.buffer); + httpTraceDump("context:%p, fd:%d, ip:%s, thread:%s, numOfFds:%d, read size:%d, raw data:\n%s", pContext, pContext->fd, + pContext->ipstr, pContext->pThread->label, pContext->pThread->numOfFds, pContext->parser.bufsize, + pContext->parser.buffer); if (!httpGetHttpMethod(pContext)) { return false; diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index 319d87d496..9276637d0e 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -76,8 +76,8 @@ int httpWriteBuf(struct HttpContext *pContext, const char *buf, int sz) { httpError("context:%p, fd:%d, ip:%s, dataSize:%d, writeSize:%d, failed to send response:\n%s", pContext, pContext->fd, pContext->ipstr, sz, writeSz, buf); } else { - httpDebug("context:%p, fd:%d, ip:%s, dataSize:%d, writeSize:%d, response:\n%s", - pContext, pContext->fd, pContext->ipstr, sz, writeSz, buf); + httpTrace("context:%p, fd:%d, ip:%s, dataSize:%d, writeSize:%d, response:\n%s", pContext, pContext->fd, + pContext->ipstr, sz, writeSz, buf); } return writeSz; @@ -99,7 +99,7 @@ int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { uint64_t srcLen = (uint64_t) (buf->lst - buf->buf); if (buf->pContext->fd <= 0) { - httpDebug("context:%p, fd:%d, ip:%s, write json body error", buf->pContext, buf->pContext->fd, buf->pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, write json body error", buf->pContext, buf->pContext->fd, buf->pContext->ipstr); buf->pContext->fd = -1; } @@ -113,11 +113,11 @@ int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { if (buf->pContext->acceptEncoding == HTTP_COMPRESS_IDENTITY) { if (buf->lst == buf->buf) { - httpDebug("context:%p, fd:%d, ip:%s, no data need dump", buf->pContext, buf->pContext->fd, buf->pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, no data need dump", buf->pContext, buf->pContext->fd, buf->pContext->ipstr); return 0; // there is no data to dump. } else { int len = sprintf(sLen, "%lx\r\n", srcLen); - httpDebug("context:%p, fd:%d, ip:%s, write body, chunkSize:%" PRIu64 ", response:\n%s", + httpTrace("context:%p, fd:%d, ip:%s, write body, chunkSize:%" PRIu64 ", response:\n%s", buf->pContext, buf->pContext->fd, buf->pContext->ipstr, srcLen, buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); remain = httpWriteBufNoTrace(buf->pContext, buf->buf, (int) srcLen); @@ -129,12 +129,12 @@ int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { if (ret == 0) { if (compressBufLen > 0) { int len = sprintf(sLen, "%x\r\n", compressBufLen); - httpDebug("context:%p, fd:%d, ip:%s, write body, chunkSize:%" PRIu64 ", compressSize:%d, last:%d, response:\n%s", + httpTrace("context:%p, fd:%d, ip:%s, write body, chunkSize:%" PRIu64 ", compressSize:%d, last:%d, response:\n%s", buf->pContext, buf->pContext->fd, buf->pContext->ipstr, srcLen, compressBufLen, isTheLast, buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); remain = httpWriteBufNoTrace(buf->pContext, (const char *) compressBuf, (int) compressBufLen); } else { - httpDebug("context:%p, fd:%d, ip:%s, last:%d, compress already dumped, response:\n%s", + httpTrace("context:%p, fd:%d, ip:%s, last:%d, compress already dumped, response:\n%s", buf->pContext, buf->pContext->fd, buf->pContext->ipstr, isTheLast, buf->buf); return 0; // there is no data to dump. } @@ -173,7 +173,7 @@ void httpWriteJsonBufHead(JsonBuf* buf) { void httpWriteJsonBufEnd(JsonBuf* buf) { if (buf->pContext->fd <= 0) { - httpDebug("context:%p, fd:%d, ip:%s, json buf fd is 0", buf->pContext, buf->pContext->fd, buf->pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, json buf fd is 0", buf->pContext, buf->pContext->fd, buf->pContext->ipstr); buf->pContext->fd = -1; } diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index cef0e80690..6c82386d81 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -66,8 +66,6 @@ void httpCleanUpConnect() { } } - tfree(pServer->pThreads); - pServer->pThreads = NULL; httpDebug("http server:%s is cleaned up", pServer->label); } diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index a93e7cd7ad..3a0998f2e8 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -95,11 +95,13 @@ void httpCleanUpSystem() { httpInfo("http server cleanup"); httpStopSystem(); + httpCleanUpConnect(); httpCleanupContexts(); httpCleanUpSessions(); - httpCleanUpConnect(); pthread_mutex_destroy(&tsHttpServer.serverMutex); - + tfree(tsHttpServer.pThreads); + tsHttpServer.pThreads = NULL; + tsHttpServer.status = HTTP_SERVER_CLOSED; } diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 2e57ad83ae..720741f089 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -119,7 +119,7 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNo int32_t size = pNode->size; taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); - uDebug("key:%s is removed from cache,total:%" PRId64 ",size:%dbytes", pNode->key, pCacheObj->totalSize, size); + uDebug("key:%s, is removed from cache, total:%" PRId64 " size:%d bytes", pNode->key, pCacheObj->totalSize, size); if (pCacheObj->freeFp) pCacheObj->freeFp(pNode->data); free(pNode); } @@ -288,14 +288,14 @@ void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, siz if (NULL != pNode) { pCacheObj->totalSize += pNode->size; - uDebug("key:%s %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", total:%" PRId64 ", size:%" PRId64 " bytes", + uDebug("key:%s, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", total:%" PRId64 ", size:%" PRId64 " bytes", key, pNode, pNode->addedTime, pNode->expiredTime, pCacheObj->totalSize, dataSize); } else { - uError("key:%s failed to added into cache, out of memory", key); + uError("key:%s, failed to added into cache, out of memory", key); } } else { // old data exists, update the node pNode = taosUpdateCacheImpl(pCacheObj, pOld, key, keyLen, pData, dataSize, duration * 1000L); - uDebug("key:%s %p exist in cache, updated", key, pNode); + uDebug("key:%s, %p exist in cache, updated", key, pNode); } __cache_unlock(pCacheObj); @@ -321,10 +321,10 @@ void *taosCacheAcquireByName(SCacheObj *pCacheObj, const char *key) { if (ptNode != NULL) { atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); - uDebug("key:%s is retrieved from cache, %p refcnt:%d", key, (*ptNode), T_REF_VAL_GET(*ptNode)); + uDebug("key:%s, is retrieved from cache, %p refcnt:%d", key, (*ptNode), T_REF_VAL_GET(*ptNode)); } else { atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); - uDebug("key:%s not in cache, retrieved failed", key); + uDebug("key:%s, not in cache, retrieved failed", key); } atomic_add_fetch_32(&pCacheObj->statistics.totalAccess, 1); @@ -350,10 +350,10 @@ void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, const char *key, uin if (ptNode != NULL) { atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); - uDebug("key:%s expireTime is updated in cache, %p refcnt:%d", key, (*ptNode), T_REF_VAL_GET(*ptNode)); + uDebug("key:%s, expireTime is updated in cache, %p refcnt:%d", key, (*ptNode), T_REF_VAL_GET(*ptNode)); } else { atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); - uDebug("key:%s not in cache, retrieved failed", key); + uDebug("key:%s, not in cache, retrieved failed", key); } atomic_add_fetch_32(&pCacheObj->statistics.totalAccess, 1); @@ -410,13 +410,13 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { SCacheDataNode *pNode = (SCacheDataNode *)((char *)(*data) - offset); if (pNode->signature != (uint64_t)pNode) { - uError("key: %p release invalid cache data", pNode); + uError("%p release invalid cache data", pNode); return; } *data = NULL; int32_t ref = T_REF_DEC(pNode); - uDebug("%p data released, refcnt:%d", pNode, ref); + uDebug("key:%s, is released, %p refcnt:%d", pNode->key, pNode, ref); if (_remove) { __cache_wr_lock(pCacheObj); @@ -501,7 +501,7 @@ void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { pNode->inTrashCan = true; pCacheObj->numOfElemsInTrash++; - uDebug("key:%s %p move to trash, numOfElem in trash:%d", pNode->key, pNode, pCacheObj->numOfElemsInTrash); + uDebug("key:%s, %p move to trash, numOfElem in trash:%d", pNode->key, pNode, pCacheObj->numOfElemsInTrash); } void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem) { @@ -549,7 +549,7 @@ void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) { } if (force || (T_REF_VAL_GET(pElem->pData) == 0)) { - uDebug("key:%s %p removed from trash. numOfElem in trash:%d", pElem->pData->key, pElem->pData, + uDebug("key:%s, %p removed from trash. numOfElem in trash:%d", pElem->pData->key, pElem->pData, pCacheObj->numOfElemsInTrash - 1); STrashElem *p = pElem; @@ -570,8 +570,11 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { while (taosHashIterNext(pIter)) { SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); // if (pNode->expiredTime <= expiredTime && T_REF_VAL_GET(pNode) <= 0) { - taosCacheReleaseNode(pCacheObj, pNode); - //} + if (T_REF_VAL_GET(pNode) <= 0) { + taosCacheReleaseNode(pCacheObj, pNode); + } else { + uDebug("key:%s, will not remove from cache, refcnt:%d", pNode->key, T_REF_VAL_GET(pNode)); + } } taosHashDestroyIter(pIter); From 9516228625c5016e5dd9abcc0c5a952a13c3f1cb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 1 Jul 2020 14:11:44 +0800 Subject: [PATCH 11/18] deadlock in rpc while taosd stopped --- src/rpc/src/rpcMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index dbdfa28a1a..3a93975022 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1054,7 +1054,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { } else { // it's a response SRpcReqContext *pContext = pConn->pContext; - rpcMsg.handle = pContext->ahandle; + rpcMsg.handle = pContext; pConn->pContext = NULL; // for UDP, port may be changed by server, the port in ipSet shall be used for cache @@ -1258,7 +1258,7 @@ static void rpcProcessConnError(void *param, void *id) { if (pContext->numOfTry >= pContext->ipSet.numOfIps) { rpcMsg.msgType = pContext->msgType+1; - rpcMsg.handle = pContext->ahandle; + rpcMsg.ahandle = pContext->ahandle; rpcMsg.code = pContext->code; rpcMsg.pCont = NULL; rpcMsg.contLen = 0; From 36d7206c390ce2a9a9635d36b1ea8bb770abc7cf Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 1 Jul 2020 06:13:09 +0000 Subject: [PATCH 12/18] rpcMsg changes --- src/rpc/src/rpcMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index dbdfa28a1a..3a93975022 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1054,7 +1054,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { } else { // it's a response SRpcReqContext *pContext = pConn->pContext; - rpcMsg.handle = pContext->ahandle; + rpcMsg.handle = pContext; pConn->pContext = NULL; // for UDP, port may be changed by server, the port in ipSet shall be used for cache @@ -1258,7 +1258,7 @@ static void rpcProcessConnError(void *param, void *id) { if (pContext->numOfTry >= pContext->ipSet.numOfIps) { rpcMsg.msgType = pContext->msgType+1; - rpcMsg.handle = pContext->ahandle; + rpcMsg.ahandle = pContext->ahandle; rpcMsg.code = pContext->code; rpcMsg.pCont = NULL; rpcMsg.contLen = 0; From b1a4752b9f28724b33a7c25f856ec94ea9d1e83e Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 1 Jul 2020 15:07:30 +0800 Subject: [PATCH 13/18] TD-803: if non-existing columns as NULL --- src/client/src/tscSQLParser.c | 1 - src/common/src/ttypes.c | 22 ++++++++++++++++++++++ src/inc/taosdef.h | 1 + src/tsdb/src/tsdbMeta.c | 5 +++++ 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 25fb2968f7..57ae8426ff 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5664,7 +5664,6 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return ret; } - //pTag->dataLen = tagVal - pTag->data; return TSDB_CODE_SUCCESS; } diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index 3c63e2deac..8ea4fe906a 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -464,6 +464,28 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) { } } +static uint8_t nullBool = TSDB_DATA_BOOL_NULL; +static uint8_t nullTinyInt = TSDB_DATA_TINYINT_NULL; +static uint16_t nullSmallInt = TSDB_DATA_SMALLINT_NULL; +static uint32_t nullInt = TSDB_DATA_INT_NULL; +static uint64_t nullBigInt = TSDB_DATA_BIGINT_NULL; +static uint32_t nullFloat = TSDB_DATA_FLOAT_NULL; +static uint64_t nullDouble = TSDB_DATA_DOUBLE_NULL; +static struct { + tstr str; + int32_t pad; +} nullBinary = {.str = {.len = 1}, .pad = 0}, nullNchar = {.str = {.len = 4}, .pad = 0}; + +static void *nullValues[] = { + &nullBool, &nullTinyInt, &nullSmallInt, &nullInt, &nullBigInt, + &nullFloat, &nullDouble, &nullBinary, &nullBigInt, &nullNchar, +}; + +void *getNullValue(int32_t type) { + assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_NCHAR); + return nullValues[type - 1]; +} + void assignVal(char *val, const char *src, int32_t len, int32_t type) { switch (type) { case TSDB_DATA_TYPE_INT: { diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index ecf78edfd5..76ca99c9ad 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -165,6 +165,7 @@ bool isNull(const char *val, int32_t type); void setVardataNull(char* val, int32_t type); void setNull(char *val, int32_t type, int32_t bytes); void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems); +void* getNullValue(int32_t type); void assignVal(char *val, const char *src, int32_t len, int32_t type); void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 7851329c47..4f331d8131 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -604,6 +604,10 @@ static char *getTagIndexKey(const void *pData) { STSchema *pSchema = tsdbGetTableTagSchema(pTable); STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN); void * res = tdGetKVRowValOfCol(pTable->tagVal, pCol->colId); + if (res == NULL) { + // treat the column as NULL if we cannot find it + res = getNullValue(pCol->type); + } return res; } @@ -849,6 +853,7 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { STable *pSTable = tsdbGetTableByUid(pMeta, TABLE_SUID(pTable)); ASSERT(pSTable != NULL); + printf("pTable = %p, pSTable = %p\n", pTable, pSTable); pTable->pSuper = pSTable; int32_t level = 0; From f4492a175fb4fcc20a9b0ae9b04cef5c6a02dc60 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 1 Jul 2020 15:35:39 +0800 Subject: [PATCH 14/18] [TD-798] --- src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 1 + src/dnode/src/dnodeMain.c | 5 +++++ src/os/linux/inc/os.h | 3 +++ src/os/linux/src/linuxPlatform.c | 29 +++++++++++++++++++++++++++++ src/util/inc/tutil.h | 1 + src/util/src/tutil.c | 10 ++++++++++ src/vnode/src/vnodeMain.c | 1 + 8 files changed, 51 insertions(+) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index af1d7dd441..e7927605cb 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -117,6 +117,7 @@ extern char tsDataDir[]; extern char tsLogDir[]; extern char tsScriptDir[]; extern int64_t tsMsPerDay[3]; +extern char tsVnodeBakDir[]; // system info extern char tsOsName[]; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 0fb63c0476..225f12a210 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -153,6 +153,7 @@ char tsDnodeDir[TSDB_FILENAME_LEN] = {0}; char tsMnodeDir[TSDB_FILENAME_LEN] = {0}; char tsDataDir[TSDB_FILENAME_LEN] = "/var/lib/taos"; char tsScriptDir[TSDB_FILENAME_LEN] = "/etc/taos"; +char tsVnodeBakDir[TSDB_FILENAME_LEN] = {0}; /* * minimum scale for whole system, millisecond by default diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 2ec87cbdd4..8d1ae0a50e 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -171,6 +171,7 @@ static int32_t dnodeInitStorage() { sprintf(tsMnodeDir, "%s/mnode", tsDataDir); sprintf(tsVnodeDir, "%s/vnode", tsDataDir); sprintf(tsDnodeDir, "%s/dnode", tsDataDir); + sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir); //TODO(dengyihao): no need to init here if (dnodeCreateDir(tsMnodeDir) < 0) { @@ -186,6 +187,10 @@ static int32_t dnodeInitStorage() { dError("failed to create dir: %s, reason: %s", tsDnodeDir, strerror(errno)); return -1; } + if (dnodeCreateDir(tsVnodeBakDir) < 0) { + dError("failed to create dir: %s, reason: %s", tsVnodeBakDir, strerror(errno)); + return -1; + } dnodeCheckDataDirOpenned(tsDnodeDir); diff --git a/src/os/linux/inc/os.h b/src/os/linux/inc/os.h index 35d2236001..58e255f7bc 100644 --- a/src/os/linux/inc/os.h +++ b/src/os/linux/inc/os.h @@ -236,6 +236,9 @@ void taosSetCoreDump(); void taosBlockSIGPIPE(); +int tSystem(const char * cmd) ; + + #ifdef _ALPINE typedef int(*__compar_fn_t)(const void *, const void *); void error (int, int, const char *); diff --git a/src/os/linux/src/linuxPlatform.c b/src/os/linux/src/linuxPlatform.c index a8ecf43c46..9a38c98f81 100644 --- a/src/os/linux/src/linuxPlatform.c +++ b/src/os/linux/src/linuxPlatform.c @@ -241,3 +241,32 @@ void taosBlockSIGPIPE() { uError("failed to block SIGPIPE"); } } + +int tSystem(const char * cmd) +{ + FILE * fp; + int res; + char buf[1024]; + if (cmd == NULL) { + uError("tSystem cmd is NULL!\n"); + return -1; + } + + if ((fp = popen(cmd, "r") ) == NULL) { + uError("popen cmd:%s error: %s/n", cmd, strerror(errno)); + return -1; + } else { + while(fgets(buf, sizeof(buf), fp)) { + uDebug("popen result:%s", buf); + } + + if ((res = pclose(fp)) == -1) { + uError("close popen file pointer fp error!\n"); + } else { + uDebug("popen res is :%d\n", res); + } + + return res; + } +} + diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index 949e9eb109..1ba57bbaaa 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -184,6 +184,7 @@ uint32_t ip2uint(const char *const ip_addr); void taosRemoveDir(char *rootDir); int tmkdir(const char *pathname, mode_t mode); +void taosMvDir(char* destDir, char *srcDir); #define TAOS_ALLOC_MODE_DEFAULT 0 #define TAOS_ALLOC_MODE_RANDOM_FAIL 1 diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index cc720eb5f7..9c5bffef95 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -799,3 +799,13 @@ int tmkdir(const char *path, mode_t mode) { if (code < 0 && errno == EEXIST) code = 0; return code; } + +void taosMvDir(char* destDir, char *srcDir) { + char shellCmd[1024+1] = {0}; + + //(void)snprintf(shellCmd, 1024, "cp -rf %s %s", srcDir, destDir); + (void)snprintf(shellCmd, 1024, "mv %s %s", srcDir, destDir); + tSystem(shellCmd); + uInfo("shell cmd:%s is executed", shellCmd); +} + diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index cbd7953072..f71f6adefb 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -347,6 +347,7 @@ void vnodeRelease(void *pVnodeRaw) { if (pVnode->status == TAOS_VN_STATUS_DELETING) { char rootDir[TSDB_FILENAME_LEN] = {0}; sprintf(rootDir, "%s/vnode%d", tsVnodeDir, vgId); + taosMvDir(tsVnodeBakDir, rootDir); taosRemoveDir(rootDir); } From c9600717ad4dcb7bbaabe041506d4b978ab1516d Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 1 Jul 2020 15:58:59 +0800 Subject: [PATCH 15/18] fix compile error and reuse getNullValue --- src/common/src/ttypes.c | 7 ++++--- src/cq/src/cqMain.c | 27 +++++---------------------- 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index 8ea4fe906a..eff25e8c93 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -471,10 +471,11 @@ static uint32_t nullInt = TSDB_DATA_INT_NULL; static uint64_t nullBigInt = TSDB_DATA_BIGINT_NULL; static uint32_t nullFloat = TSDB_DATA_FLOAT_NULL; static uint64_t nullDouble = TSDB_DATA_DOUBLE_NULL; -static struct { + +static union { tstr str; - int32_t pad; -} nullBinary = {.str = {.len = 1}, .pad = 0}, nullNchar = {.str = {.len = 4}, .pad = 0}; + char pad[sizeof(tstr) + 4]; +} nullBinary = {.str = {.len = 1}}, nullNchar = {.str = {.len = 4}}; static void *nullValues[] = { &nullBool, &nullTinyInt, &nullSmallInt, &nullInt, &nullBigInt, diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 098d69fcb2..3e7e8525ef 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -256,30 +256,13 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { SDataRow trow = (SDataRow)pBlk->data; tdInitDataRow(trow, pSchema); - union { - char buf[sizeof(int64_t)]; - tstr str; - } nullVal; - for (int32_t i = 0; i < pSchema->numOfCols; i++) { STColumn *c = pSchema->columns + i; - char* val = (char*)row[i]; - if (IS_VAR_DATA_TYPE(c->type)) { - if (val == NULL) { - val = nullVal.buf; - if (c->type == TSDB_DATA_TYPE_BINARY) { - setNull(nullVal.str.data, TSDB_DATA_TYPE_BINARY, 1); - nullVal.str.len = 1; - } else { - setNull(nullVal.str.data, TSDB_DATA_TYPE_NCHAR, 4); - nullVal.str.len = 4; - } - } else { - val -= sizeof(VarDataLenT); - } - } else if (val == NULL) { - val = nullVal.buf; - setNull(val, c->type, c->bytes); + void* val = row[i]; + if (val == NULL) { + val = getNullValue(c->type); + } else if (IS_VAR_DATA_TYPE(c->type)) { + val = ((char*)val) - sizeof(VarDataLenT); } tdAppendColVal(trow, val, c->type, c->bytes, c->offset); } From 2ed7cdd35f7a7cf637c36ce94c50fef221dc2eaa Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 1 Jul 2020 16:57:11 +0800 Subject: [PATCH 16/18] remove debug printf --- src/tsdb/src/tsdbMeta.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 4f331d8131..dafc7dbb1b 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -853,7 +853,6 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { STable *pSTable = tsdbGetTableByUid(pMeta, TABLE_SUID(pTable)); ASSERT(pSTable != NULL); - printf("pTable = %p, pSTable = %p\n", pTable, pSTable); pTable->pSuper = pSTable; int32_t level = 0; From f90f5d5fb337adfa78067c56d9f29209e2d86ba9 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 1 Jul 2020 09:28:11 +0000 Subject: [PATCH 17/18] rand() is changed to random() --- src/rpc/src/rpcMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 3a93975022..facfb79352 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -641,7 +641,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) { pConn->pRpc = pRpc; pConn->sid = sid; - pConn->tranId = (uint16_t)(rand() & 0xFFFF); + pConn->tranId = (uint16_t)(random() & 0xFFFF); pConn->ownId = htonl(pConn->sid); pConn->linkUid = (uint32_t)((int64_t)pConn + (int64_t)getpid()); pConn->spi = pRpc->spi; From 9f4ce03e8409e3e0d6c0892eb11464d01f69703f Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 1 Jul 2020 17:41:31 +0800 Subject: [PATCH 18/18] sort KVRow by colId --- src/client/src/tscParseInsert.c | 11 ++++++++--- src/client/src/tscSQLParser.c | 12 +++++++++--- src/common/inc/tdataformat.h | 1 + src/common/src/tdataformat.c | 16 ++++++++++++++++ 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index fd2a9f5a8c..4400ca2c25 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -918,6 +918,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { char tagVal[TSDB_MAX_TAGS_LEN]; code = tsParseOneColumnData(pSchema, &sToken, tagVal, pCmd->payload, &sql, false, tinfo.precision); if (code != TSDB_CODE_SUCCESS) { + tdDestroyKVRowBuilder(&kvRowBuilder); return code; } @@ -925,10 +926,14 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - pTag->dataLen = kvRowLen(row); - memcpy(pTag->data, row, pTag->dataLen); - free(row); tdDestroyKVRowBuilder(&kvRowBuilder); + if (row == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + tdSortKVRowByColIdx(row); + pTag->dataLen = kvRowLen(row); + kvRowCpy(pTag->data, row); + free(row); index = 0; sToken = tStrGetToken(sql, &index, false, 0, NULL); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 57ae8426ff..9f557f5529 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5634,6 +5634,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { // validate the length of binary if (pList->a[i].pVar.nLen + VARSTR_HEADER_SIZE > pSchema->bytes) { + tdDestroyKVRowBuilder(&kvRowBuilder); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } @@ -5641,6 +5642,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { char tagVal[TSDB_MAX_TAGS_LEN]; ret = tVariantDump(&(pList->a[i].pVar), tagVal, pSchema->type, true); if (ret != TSDB_CODE_SUCCESS) { + tdDestroyKVRowBuilder(&kvRowBuilder); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } @@ -5648,10 +5650,14 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { } SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - pTag->dataLen = kvRowLen(row); - memcpy(pTag->data, row, pTag->dataLen); - free(row); tdDestroyKVRowBuilder(&kvRowBuilder); + if (row == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + tdSortKVRowByColIdx(row); + pTag->dataLen = kvRowLen(row); + kvRowCpy(pTag->data, row); + free(row); // table name if (tscValidateName(&pInfo->pCreateTableInfo->name) != TSDB_CODE_SUCCESS) { diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index da2f07da04..baa212d8b7 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -290,6 +290,7 @@ SKVRow tdKVRowDup(SKVRow row); int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value); int tdEncodeKVRow(void **buf, SKVRow row); void * tdDecodeKVRow(void *buf, SKVRow *row); +void tdSortKVRowByColIdx(SKVRow row); static FORCE_INLINE int comparTagId(const void *key1, const void *key2) { if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) { diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index d7b2af8706..e5cbcfd143 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -515,6 +515,22 @@ SKVRow tdKVRowDup(SKVRow row) { return trow; } +static int compareColIdx(const void* a, const void* b) { + const SColIdx* x = (const SColIdx*)a; + const SColIdx* y = (const SColIdx*)b; + if (x->colId > y->colId) { + return 1; + } + if (x->colId < y->colId) { + return -1; + } + return 0; +} + +void tdSortKVRowByColIdx(SKVRow row) { + qsort(kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), compareColIdx); +} + int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { SColIdx *pColIdx = NULL; SKVRow row = *orow;