From 67cf254dde75eadedc38cf58f5bf5c20190069d7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 30 Jun 2020 14:27:56 +0800 Subject: [PATCH 01/31] fix maxTable limitation --- src/tsdb/src/tsdbMain.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 6c6b313073..2ccc962081 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -415,11 +415,11 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { // Check maxTables if (pCfg->maxTables == -1) { - pCfg->maxTables = TSDB_DEFAULT_TABLES; + pCfg->maxTables = TSDB_DEFAULT_TABLES+1; } else { - if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) { + if (pCfg->maxTables - 1 < TSDB_MIN_TABLES || pCfg->maxTables - 1 > TSDB_MAX_TABLES + 1) { tsdbError("vgId:%d invalid maxTables configuration! maxTables %d TSDB_MIN_TABLES %d TSDB_MAX_TABLES %d", - pCfg->tsdbId, pCfg->maxTables, TSDB_MIN_TABLES, TSDB_MAX_TABLES); + pCfg->tsdbId, pCfg->maxTables - 1, TSDB_MIN_TABLES, TSDB_MAX_TABLES); goto _err; } } From 12346fc4c30a9e10581be7f886046cb2c66e242d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 30 Jun 2020 16:11:14 +0800 Subject: [PATCH 02/31] fix invalid read after query --- src/vnode/src/vnodeRead.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 0c08c77e32..34d3be1489 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -188,6 +188,5 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } } - vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed", pVnode->vgId, pQInfo); return code; } From 65f3a7ef8dfb3d5cfc9d5242aa25c43b9c02d9dd Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 30 Jun 2020 11:26:10 +0000 Subject: [PATCH 03/31] change the rpcMsg definition --- src/client/src/tscServer.c | 7 ++++--- src/inc/trpc.h | 8 ++++---- src/mnode/src/mnodeDnode.c | 2 +- src/mnode/src/mnodeTable.c | 18 +++++++++--------- src/mnode/src/mnodeVgroup.c | 18 +++++++++--------- src/rpc/src/rpcMain.c | 7 ++++--- src/rpc/test/rclient.c | 6 +++--- 7 files changed, 34 insertions(+), 32 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 470a829805..0ffe50c8f4 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -191,7 +191,8 @@ int tscSendMsgToServer(SSqlObj *pSql) { .msgType = pSql->cmd.msgType, .pCont = pMsg, .contLen = pSql->cmd.payloadLen, - .handle = pSql, + .ahandle = pSql, + .handle = &pSql->pRpcCtx, .code = 0 }; @@ -199,12 +200,12 @@ int tscSendMsgToServer(SSqlObj *pSql) { // Otherwise, the pSql object may have been released already during the response function, which is // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely // cause crash. - /*pSql->pRpcCtx = */rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg); + rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg); return TSDB_CODE_SUCCESS; } void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { - SSqlObj *pSql = (SSqlObj *)rpcMsg->handle; + SSqlObj *pSql = (SSqlObj *)rpcMsg->ahandle; if (pSql == NULL || pSql->signature != pSql) { tscError("%p sql is already released", pSql); return; diff --git a/src/inc/trpc.h b/src/inc/trpc.h index 30e0f9eee1..59362d669c 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -47,8 +47,8 @@ typedef struct SRpcMsg { void *pCont; int contLen; int32_t code; - void *handle; - void *ahandle; //app handle set by client, for debug purpose + void *handle; // rpc handle returned to app + void *ahandle; // app handle set by client } SRpcMsg; typedef struct SRpcInit { @@ -78,11 +78,11 @@ void rpcClose(void *); void *rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); void *rpcReallocCont(void *ptr, int contLen); -void *rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg); +void *rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg); void rpcSendResponse(const SRpcMsg *pMsg); void rpcSendRedirectRsp(void *pConn, const SRpcIpSet *pIpSet); int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); -void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, const SRpcMsg *pReq, SRpcMsg *pRsp); +void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pReq, SRpcMsg *pRsp); int rpcReportProgress(void *pConn, char *pCont, int contLen); void rpcCancelRequest(void *pContext); diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index dbe9185d8e..7e7c12cf08 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -264,7 +264,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { strcpy(pMdCfgDnode->config, pCmCfgDnode->config); SRpcMsg rpcMdCfgDnodeMsg = { - .handle = 0, + .ahandle = 0, .code = 0, .msgType = TSDB_MSG_TYPE_MD_CONFIG_DNODE, .pCont = pMdCfgDnode, diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 7dc0c5dfa5..1f1035887c 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1573,7 +1573,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup); SRpcMsg rpcMsg = { - .handle = pMsg, + .ahandle = pMsg, .pCont = pMDCreate, .contLen = htonl(pMDCreate->contLen), .code = 0, @@ -1750,7 +1750,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { mInfo("app:%p:%p, table:%s, send drop ctable msg", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); SRpcMsg rpcMsg = { - .handle = pMsg, + .ahandle = pMsg, .pCont = pDrop, .contLen = sizeof(SMDDropTableMsg), .code = 0, @@ -1798,7 +1798,7 @@ static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) { SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup); SRpcMsg rpcMsg = { - .handle = pMsg, + .ahandle = pMsg, .pCont = pMDCreate, .contLen = htonl(pMDCreate->contLen), .code = 0, @@ -2137,9 +2137,9 @@ static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) { // handle drop child response static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { - if (rpcMsg->handle == NULL) return; + if (rpcMsg->ahandle == NULL) return; - SMnodeMsg *mnodeMsg = rpcMsg->handle; + SMnodeMsg *mnodeMsg = rpcMsg->ahandle; mnodeMsg->received++; SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; @@ -2188,9 +2188,9 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { * if failed, drop the table cached */ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { - if (rpcMsg->handle == NULL) return; + if (rpcMsg->ahandle == NULL) return; - SMnodeMsg *mnodeMsg = rpcMsg->handle; + SMnodeMsg *mnodeMsg = rpcMsg->ahandle; mnodeMsg->received++; SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; @@ -2231,9 +2231,9 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { } static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) { - if (rpcMsg->handle == NULL) return; + if (rpcMsg->ahandle == NULL) return; - SMnodeMsg *mnodeMsg = rpcMsg->handle; + SMnodeMsg *mnodeMsg = rpcMsg->ahandle; mnodeMsg->received++; SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index b314bfc8d0..3855de4101 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -652,7 +652,7 @@ SRpcIpSet mnodeGetIpSetFromIp(char *ep) { void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) { SMDCreateVnodeMsg *pCreate = mnodeBuildCreateVnodeMsg(pVgroup); SRpcMsg rpcMsg = { - .handle = ahandle, + .ahandle = ahandle, .pCont = pCreate, .contLen = pCreate ? sizeof(SMDCreateVnodeMsg) : 0, .code = 0, @@ -673,9 +673,9 @@ void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) { } static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { - if (rpcMsg->handle == NULL) return; + if (rpcMsg->ahandle == NULL) return; - SMnodeMsg *mnodeMsg = rpcMsg->handle; + SMnodeMsg *mnodeMsg = rpcMsg->ahandle; mnodeMsg->received++; if (rpcMsg->code == TSDB_CODE_SUCCESS) { mnodeMsg->successed++; @@ -686,7 +686,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { SVgObj *pVgroup = mnodeMsg->pVgroup; mDebug("vgId:%d, create vnode rsp received, result:%s received:%d successed:%d expected:%d, thandle:%p ahandle:%p", pVgroup->vgId, tstrerror(rpcMsg->code), mnodeMsg->received, mnodeMsg->successed, mnodeMsg->expected, - mnodeMsg->rpcMsg.handle, rpcMsg->handle); + mnodeMsg->rpcMsg.handle, rpcMsg->ahandle); if (mnodeMsg->received != mnodeMsg->expected) return; @@ -718,7 +718,7 @@ static SMDDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) { void mnodeSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) { SMDDropVnodeMsg *pDrop = mnodeBuildDropVnodeMsg(vgId); SRpcMsg rpcMsg = { - .handle = ahandle, + .ahandle = ahandle, .pCont = pDrop, .contLen = pDrop ? sizeof(SMDDropVnodeMsg) : 0, .code = 0, @@ -737,10 +737,10 @@ static void mnodeSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) { } static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg) { - mDebug("drop vnode rsp is received, handle:%p", rpcMsg->handle); - if (rpcMsg->handle == NULL) return; + mDebug("drop vnode rsp is received, handle:%p", rpcMsg->ahandle); + if (rpcMsg->ahandle == NULL) return; - SMnodeMsg *mnodeMsg = rpcMsg->handle; + SMnodeMsg *mnodeMsg = rpcMsg->ahandle; mnodeMsg->received++; if (rpcMsg->code == TSDB_CODE_SUCCESS) { mnodeMsg->code = rpcMsg->code; @@ -750,7 +750,7 @@ static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg) { SVgObj *pVgroup = mnodeMsg->pVgroup; mDebug("vgId:%d, drop vnode rsp received, result:%s received:%d successed:%d expected:%d, thandle:%p ahandle:%p", pVgroup->vgId, tstrerror(rpcMsg->code), mnodeMsg->received, mnodeMsg->successed, mnodeMsg->expected, - mnodeMsg->rpcMsg.handle, rpcMsg->handle); + mnodeMsg->rpcMsg.handle, rpcMsg->ahandle); if (mnodeMsg->received != mnodeMsg->expected) return; diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index a59024f8db..26ea93416d 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -354,13 +354,13 @@ void *rpcReallocCont(void *ptr, int contLen) { return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); } -void *rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg) { +void *rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg) { SRpcInfo *pRpc = (SRpcInfo *)shandle; SRpcReqContext *pContext; int contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); pContext = (SRpcReqContext *) (pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); - pContext->ahandle = pMsg->handle; + pContext->ahandle = pMsg->ahandle; pContext->pRpc = (SRpcInfo *)shandle; pContext->ipSet = *pIpSet; pContext->contLen = contLen; @@ -380,6 +380,7 @@ void *rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg || type == TSDB_MSG_TYPE_CM_SHOW ) pContext->connType = RPC_CONN_TCPC; + if (pMsg->handle) *((void **)pMsg->handle) = pContext; rpcSendReqToServer(pRpc, pContext); return pContext; @@ -483,7 +484,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { return 0; } -void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, const SRpcMsg *pMsg, SRpcMsg *pRsp) { +void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { SRpcReqContext *pContext; pContext = (SRpcReqContext *) (pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); diff --git a/src/rpc/test/rclient.c b/src/rpc/test/rclient.c index 1bc64b0825..e51b54e299 100644 --- a/src/rpc/test/rclient.c +++ b/src/rpc/test/rclient.c @@ -33,7 +33,7 @@ typedef struct { } SInfo; static void processResponse(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { - SInfo *pInfo = (SInfo *)pMsg->handle; + SInfo *pInfo = (SInfo *)pMsg->ahandle; tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, pMsg->code); if (pIpSet) pInfo->ipSet = *pIpSet; @@ -46,7 +46,7 @@ static int tcount = 0; static void *sendRequest(void *param) { SInfo *pInfo = (SInfo *)param; - SRpcMsg rpcMsg; + SRpcMsg rpcMsg = {0}; tDebug("thread:%d, start to send request", pInfo->index); @@ -54,7 +54,7 @@ static void *sendRequest(void *param) { pInfo->num++; rpcMsg.pCont = rpcMallocCont(pInfo->msgSize); rpcMsg.contLen = pInfo->msgSize; - rpcMsg.handle = pInfo; + rpcMsg.ahandle = pInfo; rpcMsg.msgType = 1; tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); rpcSendRequest(pInfo->pRpc, &pInfo->ipSet, &rpcMsg); From 10eabebfbcd495701478337a012f46423b94ba2a Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 30 Jun 2020 11:57:08 +0000 Subject: [PATCH 04/31] change the return type --- src/inc/trpc.h | 2 +- src/rpc/src/rpcMain.c | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/inc/trpc.h b/src/inc/trpc.h index 59362d669c..d1adfb7494 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -78,7 +78,7 @@ void rpcClose(void *); void *rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); void *rpcReallocCont(void *ptr, int contLen); -void *rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg); +void rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg); void rpcSendResponse(const SRpcMsg *pMsg); void rpcSendRedirectRsp(void *pConn, const SRpcIpSet *pIpSet); int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 26ea93416d..dbdfa28a1a 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -354,7 +354,7 @@ void *rpcReallocCont(void *ptr, int contLen) { return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); } -void *rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg) { +void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg) { SRpcInfo *pRpc = (SRpcInfo *)shandle; SRpcReqContext *pContext; @@ -380,10 +380,12 @@ void *rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg) { || type == TSDB_MSG_TYPE_CM_SHOW ) pContext->connType = RPC_CONN_TCPC; + // set the handle to pContext, so app can cancel the request if (pMsg->handle) *((void **)pMsg->handle) = pContext; + rpcSendReqToServer(pRpc, pContext); - return pContext; + return; } void rpcSendResponse(const SRpcMsg *pRsp) { From c1252c2dbee0349290dda7e8aeaddaec1390d78a Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 1 Jul 2020 11:08:19 +0800 Subject: [PATCH 05/31] TD-803: use KV to transfer tags --- src/client/src/tscParseInsert.c | 43 +++++++++++++-------------------- src/client/src/tscSQLParser.c | 25 +++++++++++++------ src/tsdb/src/tsdbMeta.c | 22 +++-------------- 3 files changed, 38 insertions(+), 52 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index f0c8ffebc2..fd2a9f5a8c 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -891,11 +891,15 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return tscInvalidSQLErrMsg(pCmd->payload, "keyword TAGS expected", sToken.z); } + SKVRowBuilder kvRowBuilder = {0}; + if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + uint32_t ignoreTokenTypes = TK_LP; uint32_t numOfIgnoreToken = 1; for (int i = 0; i < spd.numOfAssignedCols; ++i) { - char * tagVal = pTag->data + spd.elems[i].offset; - int16_t colIndex = spd.elems[i].colIndex; + SSchema* pSchema = pTagSchema + spd.elems[i].colIndex; index = 0; sToken = tStrGetToken(sql, &index, true, numOfIgnoreToken, &ignoreTokenTypes); @@ -911,12 +915,21 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { sToken.n -= 2; } - code = tsParseOneColumnData(&pTagSchema[colIndex], &sToken, tagVal, pCmd->payload, &sql, false, tinfo.precision); + char tagVal[TSDB_MAX_TAGS_LEN]; + code = tsParseOneColumnData(pSchema, &sToken, tagVal, pCmd->payload, &sql, false, tinfo.precision); if (code != TSDB_CODE_SUCCESS) { return code; } + + tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); } + SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); + pTag->dataLen = kvRowLen(row); + memcpy(pTag->data, row, pTag->dataLen); + free(row); + tdDestroyKVRowBuilder(&kvRowBuilder); + index = 0; sToken = tStrGetToken(sql, &index, false, 0, NULL); sql += index; @@ -924,29 +937,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return tscInvalidSQLErrMsg(pCmd->payload, ") expected", sToken.z); } - // 2. set the null value for the columns that do not assign values - if (spd.numOfAssignedCols < spd.numOfCols) { - char *ptr = pTag->data; - - for (int32_t i = 0; i < spd.numOfCols; ++i) { - if (!spd.hasVal[i]) { // current tag column do not have any value to insert, set it to null - if (pTagSchema[i].type == TSDB_DATA_TYPE_BINARY || pTagSchema[i].type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(ptr, pTagSchema[i].type); - } else { - setNull(ptr, pTagSchema[i].type, pTagSchema[i].bytes); - } - } - - ptr += pTagSchema[i].bytes; - } - } - - // 3. calculate the actual data size of STagData - pCmd->payloadLen = sizeof(pTag->name) + sizeof(pTag->dataLen); - for (int32_t t = 0; t < numOfTags; ++t) { - pTag->dataLen += pTagSchema[t].bytes; - pCmd->payloadLen += pTagSchema[t].bytes; - } + pCmd->payloadLen = sizeof(pTag->name) + sizeof(pTag->dataLen) + pTag->dataLen; pTag->dataLen = htonl(pTag->dataLen); if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index df18d7a56b..25fb2968f7 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5623,25 +5623,36 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { SSchema* pTagSchema = tscGetTableTagSchema(pStableMeterMetaInfo->pTableMeta); STagData* pTag = &pCreateTable->usingInfo.tagdata; - char* tagVal = pTag->data; + SKVRowBuilder kvRowBuilder = {0}; + if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + int32_t ret = TSDB_CODE_SUCCESS; - for (int32_t i = 0; i < pList->nExpr; ++i) { - if (pTagSchema[i].type == TSDB_DATA_TYPE_BINARY || pTagSchema[i].type == TSDB_DATA_TYPE_NCHAR) { + SSchema* pSchema = pTagSchema + i; + if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { // validate the length of binary - if (pList->a[i].pVar.nLen + VARSTR_HEADER_SIZE > pTagSchema[i].bytes) { + if (pList->a[i].pVar.nLen + VARSTR_HEADER_SIZE > pSchema->bytes) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } - ret = tVariantDump(&(pList->a[i].pVar), tagVal, pTagSchema[i].type, true); + char tagVal[TSDB_MAX_TAGS_LEN]; + ret = tVariantDump(&(pList->a[i].pVar), tagVal, pSchema->type, true); if (ret != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } - tagVal += pTagSchema[i].bytes; + tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); } + SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); + pTag->dataLen = kvRowLen(row); + memcpy(pTag->data, row, pTag->dataLen); + free(row); + tdDestroyKVRowBuilder(&kvRowBuilder); + // table name if (tscValidateName(&pInfo->pCreateTableInfo->name) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); @@ -5653,7 +5664,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return ret; } - pTag->dataLen = tagVal - pTag->data; + //pTag->dataLen = tagVal - pTag->data; return TSDB_CODE_SUCCESS; } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 13b676ee1d..7851329c47 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -233,26 +233,10 @@ STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) { if (tsdbTableSetSName(pCfg, pMsg->superTableId, true) < 0) goto _err; if (tsdbTableSetSuperUid(pCfg, htobe64(pMsg->superTableUid)) < 0) goto _err; - // Decode tag values - if (pMsg->tagDataLen) { - int accBytes = 0; + int32_t tagDataLen = htonl(pMsg->tagDataLen); + if (tagDataLen) { char *pTagData = pMsg->data + (numOfCols + numOfTags) * sizeof(SSchema); - - SKVRowBuilder kvRowBuilder = {0}; - if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - for (int i = numOfCols; i < numOfCols + numOfTags; i++) { - if (tdAddColToKVRow(&kvRowBuilder, htons(pSchema[i].colId), pSchema[i].type, pTagData + accBytes) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - accBytes += htons(pSchema[i].bytes); - } - - tsdbTableSetTagValue(pCfg, tdGetKVRowFromBuilder(&kvRowBuilder), false); - tdDestroyKVRowBuilder(&kvRowBuilder); + tsdbTableSetTagValue(pCfg, pTagData, true); } } From 29adf9c752a0935a840d703549807a68805117d7 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 1 Jul 2020 11:17:48 +0800 Subject: [PATCH 06/31] TD-803: increase bufsize --- src/common/inc/tdataformat.h | 2 +- src/inc/taosmsg.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 1cd72eafde..da2f07da04 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -272,7 +272,7 @@ typedef struct { int16_t offset; } SColIdx; -#define TD_KV_ROW_HEAD_SIZE 2 * sizeof(int16_t) +#define TD_KV_ROW_HEAD_SIZE (2 * sizeof(int16_t)) #define kvRowLen(r) (*(int16_t *)(r)) #define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 13fa799b3f..cb25242d27 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -26,6 +26,7 @@ extern "C" { #include "taosdef.h" #include "taoserror.h" #include "trpc.h" +#include "tdataformat.h" // message type @@ -674,7 +675,7 @@ typedef struct SMultiTableMeta { typedef struct { int32_t dataLen; char name[TSDB_TABLE_ID_LEN]; - char data[TSDB_MAX_TAGS_LEN]; + char data[TSDB_MAX_TAGS_LEN + TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * TSDB_MAX_TAGS]; } STagData; /* From 06a4d67734a322a8b1c4e659d40205f97562ed10 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jul 2020 11:49:21 +0800 Subject: [PATCH 07/31] change debug as trace --- src/tsdb/src/tsdbMeta.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 13b676ee1d..ebc83d0cc2 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -557,7 +557,7 @@ void tsdbRefTable(STable *pTable) { void tsdbUnRefTable(STable *pTable) { int32_t ref = T_REF_DEC(pTable); - tsdbDebug("unref table uid:%"PRIu64", tid:%d, refCount:%d", TABLE_UID(pTable), TABLE_TID(pTable), ref); + tsdbTrace("unref table uid:%"PRIu64", tid:%d, refCount:%d", TABLE_UID(pTable), TABLE_TID(pTable), ref); if (ref == 0) { // tsdbDebug("destory table name:%s uid:%"PRIu64", tid:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable)); From 13f8b063e5090ebad8547b3defbcb9cfb560ef93 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 1 Jul 2020 11:49:41 +0800 Subject: [PATCH 08/31] fix crash when there are 2 or more binary columns --- src/client/src/tscUtil.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 259bcd4cbd..26a81c597f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -625,18 +625,31 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo return len; } +static int32_t getRowExpandSize(STableMeta* pTableMeta) { + int32_t result = TD_DATA_ROW_HEAD_SIZE; + int32_t columns = tscGetNumOfColumns(pTableMeta); + SSchema* pSchema = tscGetTableSchema(pTableMeta); + for(int32_t i = 0; i < columns; i++) { + if (IS_VAR_DATA_TYPE((pSchema + i)->type)) { + result += TYPE_BYTES[TSDB_DATA_TYPE_BINARY]; + } + } + return result; +} + int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { SSqlCmd* pCmd = &pSql->cmd; // the maximum expanded size in byte when a row-wise data is converted to SDataRow format - const int32_t MAX_EXPAND_SIZE = TD_DATA_ROW_HEAD_SIZE + TYPE_BYTES[TSDB_DATA_TYPE_BINARY]; + STableDataBlocks* pOneTableBlock = taosArrayGetP(pTableDataBlockList, 0); + int32_t expandSize = getRowExpandSize(pOneTableBlock->pTableMeta); void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); size_t total = taosArrayGetSize(pTableDataBlockList); for (int32_t i = 0; i < total; ++i) { - STableDataBlocks* pOneTableBlock = taosArrayGetP(pTableDataBlockList, i); + pOneTableBlock = taosArrayGetP(pTableDataBlockList, i); STableDataBlocks* dataBuf = NULL; int32_t ret = @@ -650,7 +663,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { } SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; - int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * MAX_EXPAND_SIZE; + int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize; if (dataBuf->nAllocSize < destSize) { while (dataBuf->nAllocSize < destSize) { @@ -678,8 +691,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { tscDebug("%p tableId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableId, pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey)); - - int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + MAX_EXPAND_SIZE); + int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + expandSize); pBlocks->tid = htonl(pBlocks->tid); pBlocks->uid = htobe64(pBlocks->uid); From e0acd602e0c4f3af87ba34cbad85ca85594705f5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 1 Jul 2020 13:31:26 +0800 Subject: [PATCH 09/31] [TD-814] invalid read while close http connect --- src/plugins/http/src/httpContext.c | 9 +++++++-- src/plugins/http/src/httpHandle.c | 24 +++++++++++------------ src/plugins/http/src/httpJson.c | 16 +++++++-------- src/plugins/http/src/httpServer.c | 2 -- src/plugins/http/src/httpSystem.c | 6 ++++-- src/util/src/tcache.c | 31 ++++++++++++++++-------------- 6 files changed, 48 insertions(+), 40 deletions(-) diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index b078be5930..b09f34b562 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -141,10 +141,15 @@ HttpContext *httpGetContext(void *ptr) { void httpReleaseContext(HttpContext *pContext) { int32_t refCount = atomic_sub_fetch_32(&pContext->refCount, 1); assert(refCount >= 0); - httpDebug("context:%p, fd:%d, is releasd, refCount:%d", pContext, pContext->fd, refCount); + httpDebug("context:%p, is releasd, refCount:%d", pContext, refCount); HttpContext **ppContext = pContext->ppContext; - taosCacheRelease(tsHttpServer.contextCache, (void **)(&ppContext), false); + if (tsHttpServer.contextCache != NULL) { + taosCacheRelease(tsHttpServer.contextCache, (void **)(&ppContext), false); + } else { + httpDebug("context:%p, won't be destroyed for cache is already released", pContext); + // httpDestroyContext((void **)(&ppContext)); + } } bool httpInitContext(HttpContext *pContext) { diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c index 758286344a..056fe425d4 100644 --- a/src/plugins/http/src/httpHandle.c +++ b/src/plugins/http/src/httpHandle.c @@ -157,7 +157,7 @@ bool httpGetHttpMethod(HttpContext* pContext) { pParser->method.pos[pParser->method.len] = 0; pParser->pLast = pSeek + 1; - httpDebug("context:%p, fd:%d, ip:%s, httpMethod:%s", pContext, pContext->fd, pContext->ipstr, pParser->method.pos); + httpTrace("context:%p, fd:%d, ip:%s, httpMethod:%s", pContext, pContext->fd, pContext->ipstr, pParser->method.pos); return true; } @@ -186,23 +186,23 @@ bool httpParseHead(HttpContext* pContext) { HttpParser* pParser = &pContext->parser; if (strncasecmp(pParser->pLast, "Content-Length: ", 16) == 0) { pParser->data.len = (int32_t)atoi(pParser->pLast + 16); - httpDebug("context:%p, fd:%d, ip:%s, Content-Length:%d", pContext, pContext->fd, pContext->ipstr, + httpTrace("context:%p, fd:%d, ip:%s, Content-Length:%d", pContext, pContext->fd, pContext->ipstr, pParser->data.len); } else if (strncasecmp(pParser->pLast, "Accept-Encoding: ", 17) == 0) { if (tsHttpEnableCompress && strstr(pParser->pLast + 17, "gzip") != NULL) { pContext->acceptEncoding = HTTP_COMPRESS_GZIP; - httpDebug("context:%p, fd:%d, ip:%s, Accept-Encoding:gzip", pContext, pContext->fd, pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, Accept-Encoding:gzip", pContext, pContext->fd, pContext->ipstr); } else { pContext->acceptEncoding = HTTP_COMPRESS_IDENTITY; - httpDebug("context:%p, fd:%d, ip:%s, Accept-Encoding:identity", pContext, pContext->fd, pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, Accept-Encoding:identity", pContext, pContext->fd, pContext->ipstr); } } else if (strncasecmp(pParser->pLast, "Content-Encoding: ", 18) == 0) { if (strstr(pParser->pLast + 18, "gzip") != NULL) { pContext->contentEncoding = HTTP_COMPRESS_GZIP; - httpDebug("context:%p, fd:%d, ip:%s, Content-Encoding:gzip", pContext, pContext->fd, pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, Content-Encoding:gzip", pContext, pContext->fd, pContext->ipstr); } else { pContext->contentEncoding = HTTP_COMPRESS_IDENTITY; - httpDebug("context:%p, fd:%d, ip:%s, Content-Encoding:identity", pContext, pContext->fd, pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, Content-Encoding:identity", pContext, pContext->fd, pContext->ipstr); } } else if (strncasecmp(pParser->pLast, "Connection: ", 12) == 0) { if (strncasecmp(pParser->pLast + 12, "Keep-Alive", 10) == 0) { @@ -210,7 +210,7 @@ bool httpParseHead(HttpContext* pContext) { } else { pContext->httpKeepAlive = HTTP_KEEPALIVE_DISABLE; } - httpDebug("context:%p, fd:%d, ip:%s, keepAlive:%d", pContext, pContext->fd, pContext->ipstr, + httpTrace("context:%p, fd:%d, ip:%s, keepAlive:%d", pContext, pContext->fd, pContext->ipstr, pContext->httpKeepAlive); } else if (strncasecmp(pParser->pLast, "Transfer-Encoding: ", 19) == 0) { if (strncasecmp(pParser->pLast + 19, "chunked", 7) == 0) { @@ -281,7 +281,7 @@ bool httpReadChunkedBody(HttpContext* pContext, HttpParser* pParser) { httpParseChunkedBody(pContext, pParser, false); return HTTP_CHECK_BODY_SUCCESS; } else { - httpDebug("context:%p, fd:%d, ip:%s, chunked body not finished, continue read", pContext, pContext->fd, pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, chunked body not finished, continue read", pContext, pContext->fd, pContext->ipstr); if (!httpReadDataImp(pContext)) { httpError("context:%p, fd:%d, ip:%s, read chunked request error", pContext, pContext->fd, pContext->ipstr); return HTTP_CHECK_BODY_ERROR; @@ -299,7 +299,7 @@ int httpReadUnChunkedBody(HttpContext* pContext, HttpParser* pParser) { httpSendErrorResp(pContext, HTTP_PARSE_BODY_ERROR); return HTTP_CHECK_BODY_ERROR; } else if (dataReadLen < pParser->data.len) { - httpDebug("context:%p, fd:%d, ip:%s, un-chunked body not finished, read size:%d dataReadLen:%d < pContext->data.len:%d, continue read", + httpTrace("context:%p, fd:%d, ip:%s, un-chunked body not finished, read size:%d dataReadLen:%d < pContext->data.len:%d, continue read", pContext, pContext->fd, pContext->ipstr, pContext->parser.bufsize, dataReadLen, pParser->data.len); return HTTP_CHECK_BODY_CONTINUE; } else { @@ -313,9 +313,9 @@ bool httpParseRequest(HttpContext* pContext) { return true; } - httpDebug("context:%p, fd:%d, ip:%s, thread:%s, numOfFds:%d, read size:%d, raw data:\n%s", - pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->pThread->numOfFds, - pContext->parser.bufsize, pContext->parser.buffer); + httpTraceDump("context:%p, fd:%d, ip:%s, thread:%s, numOfFds:%d, read size:%d, raw data:\n%s", pContext, pContext->fd, + pContext->ipstr, pContext->pThread->label, pContext->pThread->numOfFds, pContext->parser.bufsize, + pContext->parser.buffer); if (!httpGetHttpMethod(pContext)) { return false; diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index 319d87d496..9276637d0e 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -76,8 +76,8 @@ int httpWriteBuf(struct HttpContext *pContext, const char *buf, int sz) { httpError("context:%p, fd:%d, ip:%s, dataSize:%d, writeSize:%d, failed to send response:\n%s", pContext, pContext->fd, pContext->ipstr, sz, writeSz, buf); } else { - httpDebug("context:%p, fd:%d, ip:%s, dataSize:%d, writeSize:%d, response:\n%s", - pContext, pContext->fd, pContext->ipstr, sz, writeSz, buf); + httpTrace("context:%p, fd:%d, ip:%s, dataSize:%d, writeSize:%d, response:\n%s", pContext, pContext->fd, + pContext->ipstr, sz, writeSz, buf); } return writeSz; @@ -99,7 +99,7 @@ int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { uint64_t srcLen = (uint64_t) (buf->lst - buf->buf); if (buf->pContext->fd <= 0) { - httpDebug("context:%p, fd:%d, ip:%s, write json body error", buf->pContext, buf->pContext->fd, buf->pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, write json body error", buf->pContext, buf->pContext->fd, buf->pContext->ipstr); buf->pContext->fd = -1; } @@ -113,11 +113,11 @@ int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { if (buf->pContext->acceptEncoding == HTTP_COMPRESS_IDENTITY) { if (buf->lst == buf->buf) { - httpDebug("context:%p, fd:%d, ip:%s, no data need dump", buf->pContext, buf->pContext->fd, buf->pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, no data need dump", buf->pContext, buf->pContext->fd, buf->pContext->ipstr); return 0; // there is no data to dump. } else { int len = sprintf(sLen, "%lx\r\n", srcLen); - httpDebug("context:%p, fd:%d, ip:%s, write body, chunkSize:%" PRIu64 ", response:\n%s", + httpTrace("context:%p, fd:%d, ip:%s, write body, chunkSize:%" PRIu64 ", response:\n%s", buf->pContext, buf->pContext->fd, buf->pContext->ipstr, srcLen, buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); remain = httpWriteBufNoTrace(buf->pContext, buf->buf, (int) srcLen); @@ -129,12 +129,12 @@ int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { if (ret == 0) { if (compressBufLen > 0) { int len = sprintf(sLen, "%x\r\n", compressBufLen); - httpDebug("context:%p, fd:%d, ip:%s, write body, chunkSize:%" PRIu64 ", compressSize:%d, last:%d, response:\n%s", + httpTrace("context:%p, fd:%d, ip:%s, write body, chunkSize:%" PRIu64 ", compressSize:%d, last:%d, response:\n%s", buf->pContext, buf->pContext->fd, buf->pContext->ipstr, srcLen, compressBufLen, isTheLast, buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); remain = httpWriteBufNoTrace(buf->pContext, (const char *) compressBuf, (int) compressBufLen); } else { - httpDebug("context:%p, fd:%d, ip:%s, last:%d, compress already dumped, response:\n%s", + httpTrace("context:%p, fd:%d, ip:%s, last:%d, compress already dumped, response:\n%s", buf->pContext, buf->pContext->fd, buf->pContext->ipstr, isTheLast, buf->buf); return 0; // there is no data to dump. } @@ -173,7 +173,7 @@ void httpWriteJsonBufHead(JsonBuf* buf) { void httpWriteJsonBufEnd(JsonBuf* buf) { if (buf->pContext->fd <= 0) { - httpDebug("context:%p, fd:%d, ip:%s, json buf fd is 0", buf->pContext, buf->pContext->fd, buf->pContext->ipstr); + httpTrace("context:%p, fd:%d, ip:%s, json buf fd is 0", buf->pContext, buf->pContext->fd, buf->pContext->ipstr); buf->pContext->fd = -1; } diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index cef0e80690..6c82386d81 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -66,8 +66,6 @@ void httpCleanUpConnect() { } } - tfree(pServer->pThreads); - pServer->pThreads = NULL; httpDebug("http server:%s is cleaned up", pServer->label); } diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index a93e7cd7ad..3a0998f2e8 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -95,11 +95,13 @@ void httpCleanUpSystem() { httpInfo("http server cleanup"); httpStopSystem(); + httpCleanUpConnect(); httpCleanupContexts(); httpCleanUpSessions(); - httpCleanUpConnect(); pthread_mutex_destroy(&tsHttpServer.serverMutex); - + tfree(tsHttpServer.pThreads); + tsHttpServer.pThreads = NULL; + tsHttpServer.status = HTTP_SERVER_CLOSED; } diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 2e57ad83ae..720741f089 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -119,7 +119,7 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNo int32_t size = pNode->size; taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); - uDebug("key:%s is removed from cache,total:%" PRId64 ",size:%dbytes", pNode->key, pCacheObj->totalSize, size); + uDebug("key:%s, is removed from cache, total:%" PRId64 " size:%d bytes", pNode->key, pCacheObj->totalSize, size); if (pCacheObj->freeFp) pCacheObj->freeFp(pNode->data); free(pNode); } @@ -288,14 +288,14 @@ void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, siz if (NULL != pNode) { pCacheObj->totalSize += pNode->size; - uDebug("key:%s %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", total:%" PRId64 ", size:%" PRId64 " bytes", + uDebug("key:%s, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", total:%" PRId64 ", size:%" PRId64 " bytes", key, pNode, pNode->addedTime, pNode->expiredTime, pCacheObj->totalSize, dataSize); } else { - uError("key:%s failed to added into cache, out of memory", key); + uError("key:%s, failed to added into cache, out of memory", key); } } else { // old data exists, update the node pNode = taosUpdateCacheImpl(pCacheObj, pOld, key, keyLen, pData, dataSize, duration * 1000L); - uDebug("key:%s %p exist in cache, updated", key, pNode); + uDebug("key:%s, %p exist in cache, updated", key, pNode); } __cache_unlock(pCacheObj); @@ -321,10 +321,10 @@ void *taosCacheAcquireByName(SCacheObj *pCacheObj, const char *key) { if (ptNode != NULL) { atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); - uDebug("key:%s is retrieved from cache, %p refcnt:%d", key, (*ptNode), T_REF_VAL_GET(*ptNode)); + uDebug("key:%s, is retrieved from cache, %p refcnt:%d", key, (*ptNode), T_REF_VAL_GET(*ptNode)); } else { atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); - uDebug("key:%s not in cache, retrieved failed", key); + uDebug("key:%s, not in cache, retrieved failed", key); } atomic_add_fetch_32(&pCacheObj->statistics.totalAccess, 1); @@ -350,10 +350,10 @@ void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, const char *key, uin if (ptNode != NULL) { atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); - uDebug("key:%s expireTime is updated in cache, %p refcnt:%d", key, (*ptNode), T_REF_VAL_GET(*ptNode)); + uDebug("key:%s, expireTime is updated in cache, %p refcnt:%d", key, (*ptNode), T_REF_VAL_GET(*ptNode)); } else { atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); - uDebug("key:%s not in cache, retrieved failed", key); + uDebug("key:%s, not in cache, retrieved failed", key); } atomic_add_fetch_32(&pCacheObj->statistics.totalAccess, 1); @@ -410,13 +410,13 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { SCacheDataNode *pNode = (SCacheDataNode *)((char *)(*data) - offset); if (pNode->signature != (uint64_t)pNode) { - uError("key: %p release invalid cache data", pNode); + uError("%p release invalid cache data", pNode); return; } *data = NULL; int32_t ref = T_REF_DEC(pNode); - uDebug("%p data released, refcnt:%d", pNode, ref); + uDebug("key:%s, is released, %p refcnt:%d", pNode->key, pNode, ref); if (_remove) { __cache_wr_lock(pCacheObj); @@ -501,7 +501,7 @@ void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { pNode->inTrashCan = true; pCacheObj->numOfElemsInTrash++; - uDebug("key:%s %p move to trash, numOfElem in trash:%d", pNode->key, pNode, pCacheObj->numOfElemsInTrash); + uDebug("key:%s, %p move to trash, numOfElem in trash:%d", pNode->key, pNode, pCacheObj->numOfElemsInTrash); } void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem) { @@ -549,7 +549,7 @@ void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) { } if (force || (T_REF_VAL_GET(pElem->pData) == 0)) { - uDebug("key:%s %p removed from trash. numOfElem in trash:%d", pElem->pData->key, pElem->pData, + uDebug("key:%s, %p removed from trash. numOfElem in trash:%d", pElem->pData->key, pElem->pData, pCacheObj->numOfElemsInTrash - 1); STrashElem *p = pElem; @@ -570,8 +570,11 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { while (taosHashIterNext(pIter)) { SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); // if (pNode->expiredTime <= expiredTime && T_REF_VAL_GET(pNode) <= 0) { - taosCacheReleaseNode(pCacheObj, pNode); - //} + if (T_REF_VAL_GET(pNode) <= 0) { + taosCacheReleaseNode(pCacheObj, pNode); + } else { + uDebug("key:%s, will not remove from cache, refcnt:%d", pNode->key, T_REF_VAL_GET(pNode)); + } } taosHashDestroyIter(pIter); From 9516228625c5016e5dd9abcc0c5a952a13c3f1cb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 1 Jul 2020 14:11:44 +0800 Subject: [PATCH 10/31] deadlock in rpc while taosd stopped --- src/rpc/src/rpcMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index dbdfa28a1a..3a93975022 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1054,7 +1054,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { } else { // it's a response SRpcReqContext *pContext = pConn->pContext; - rpcMsg.handle = pContext->ahandle; + rpcMsg.handle = pContext; pConn->pContext = NULL; // for UDP, port may be changed by server, the port in ipSet shall be used for cache @@ -1258,7 +1258,7 @@ static void rpcProcessConnError(void *param, void *id) { if (pContext->numOfTry >= pContext->ipSet.numOfIps) { rpcMsg.msgType = pContext->msgType+1; - rpcMsg.handle = pContext->ahandle; + rpcMsg.ahandle = pContext->ahandle; rpcMsg.code = pContext->code; rpcMsg.pCont = NULL; rpcMsg.contLen = 0; From 36d7206c390ce2a9a9635d36b1ea8bb770abc7cf Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 1 Jul 2020 06:13:09 +0000 Subject: [PATCH 11/31] rpcMsg changes --- src/rpc/src/rpcMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index dbdfa28a1a..3a93975022 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1054,7 +1054,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { } else { // it's a response SRpcReqContext *pContext = pConn->pContext; - rpcMsg.handle = pContext->ahandle; + rpcMsg.handle = pContext; pConn->pContext = NULL; // for UDP, port may be changed by server, the port in ipSet shall be used for cache @@ -1258,7 +1258,7 @@ static void rpcProcessConnError(void *param, void *id) { if (pContext->numOfTry >= pContext->ipSet.numOfIps) { rpcMsg.msgType = pContext->msgType+1; - rpcMsg.handle = pContext->ahandle; + rpcMsg.ahandle = pContext->ahandle; rpcMsg.code = pContext->code; rpcMsg.pCont = NULL; rpcMsg.contLen = 0; From b1a4752b9f28724b33a7c25f856ec94ea9d1e83e Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 1 Jul 2020 15:07:30 +0800 Subject: [PATCH 12/31] TD-803: if non-existing columns as NULL --- src/client/src/tscSQLParser.c | 1 - src/common/src/ttypes.c | 22 ++++++++++++++++++++++ src/inc/taosdef.h | 1 + src/tsdb/src/tsdbMeta.c | 5 +++++ 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 25fb2968f7..57ae8426ff 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5664,7 +5664,6 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return ret; } - //pTag->dataLen = tagVal - pTag->data; return TSDB_CODE_SUCCESS; } diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index 3c63e2deac..8ea4fe906a 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -464,6 +464,28 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) { } } +static uint8_t nullBool = TSDB_DATA_BOOL_NULL; +static uint8_t nullTinyInt = TSDB_DATA_TINYINT_NULL; +static uint16_t nullSmallInt = TSDB_DATA_SMALLINT_NULL; +static uint32_t nullInt = TSDB_DATA_INT_NULL; +static uint64_t nullBigInt = TSDB_DATA_BIGINT_NULL; +static uint32_t nullFloat = TSDB_DATA_FLOAT_NULL; +static uint64_t nullDouble = TSDB_DATA_DOUBLE_NULL; +static struct { + tstr str; + int32_t pad; +} nullBinary = {.str = {.len = 1}, .pad = 0}, nullNchar = {.str = {.len = 4}, .pad = 0}; + +static void *nullValues[] = { + &nullBool, &nullTinyInt, &nullSmallInt, &nullInt, &nullBigInt, + &nullFloat, &nullDouble, &nullBinary, &nullBigInt, &nullNchar, +}; + +void *getNullValue(int32_t type) { + assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_NCHAR); + return nullValues[type - 1]; +} + void assignVal(char *val, const char *src, int32_t len, int32_t type) { switch (type) { case TSDB_DATA_TYPE_INT: { diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index ecf78edfd5..76ca99c9ad 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -165,6 +165,7 @@ bool isNull(const char *val, int32_t type); void setVardataNull(char* val, int32_t type); void setNull(char *val, int32_t type, int32_t bytes); void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems); +void* getNullValue(int32_t type); void assignVal(char *val, const char *src, int32_t len, int32_t type); void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 7851329c47..4f331d8131 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -604,6 +604,10 @@ static char *getTagIndexKey(const void *pData) { STSchema *pSchema = tsdbGetTableTagSchema(pTable); STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN); void * res = tdGetKVRowValOfCol(pTable->tagVal, pCol->colId); + if (res == NULL) { + // treat the column as NULL if we cannot find it + res = getNullValue(pCol->type); + } return res; } @@ -849,6 +853,7 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { STable *pSTable = tsdbGetTableByUid(pMeta, TABLE_SUID(pTable)); ASSERT(pSTable != NULL); + printf("pTable = %p, pSTable = %p\n", pTable, pSTable); pTable->pSuper = pSTable; int32_t level = 0; From f4492a175fb4fcc20a9b0ae9b04cef5c6a02dc60 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 1 Jul 2020 15:35:39 +0800 Subject: [PATCH 13/31] [TD-798] --- src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 1 + src/dnode/src/dnodeMain.c | 5 +++++ src/os/linux/inc/os.h | 3 +++ src/os/linux/src/linuxPlatform.c | 29 +++++++++++++++++++++++++++++ src/util/inc/tutil.h | 1 + src/util/src/tutil.c | 10 ++++++++++ src/vnode/src/vnodeMain.c | 1 + 8 files changed, 51 insertions(+) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index af1d7dd441..e7927605cb 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -117,6 +117,7 @@ extern char tsDataDir[]; extern char tsLogDir[]; extern char tsScriptDir[]; extern int64_t tsMsPerDay[3]; +extern char tsVnodeBakDir[]; // system info extern char tsOsName[]; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 0fb63c0476..225f12a210 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -153,6 +153,7 @@ char tsDnodeDir[TSDB_FILENAME_LEN] = {0}; char tsMnodeDir[TSDB_FILENAME_LEN] = {0}; char tsDataDir[TSDB_FILENAME_LEN] = "/var/lib/taos"; char tsScriptDir[TSDB_FILENAME_LEN] = "/etc/taos"; +char tsVnodeBakDir[TSDB_FILENAME_LEN] = {0}; /* * minimum scale for whole system, millisecond by default diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 2ec87cbdd4..8d1ae0a50e 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -171,6 +171,7 @@ static int32_t dnodeInitStorage() { sprintf(tsMnodeDir, "%s/mnode", tsDataDir); sprintf(tsVnodeDir, "%s/vnode", tsDataDir); sprintf(tsDnodeDir, "%s/dnode", tsDataDir); + sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir); //TODO(dengyihao): no need to init here if (dnodeCreateDir(tsMnodeDir) < 0) { @@ -186,6 +187,10 @@ static int32_t dnodeInitStorage() { dError("failed to create dir: %s, reason: %s", tsDnodeDir, strerror(errno)); return -1; } + if (dnodeCreateDir(tsVnodeBakDir) < 0) { + dError("failed to create dir: %s, reason: %s", tsVnodeBakDir, strerror(errno)); + return -1; + } dnodeCheckDataDirOpenned(tsDnodeDir); diff --git a/src/os/linux/inc/os.h b/src/os/linux/inc/os.h index 35d2236001..58e255f7bc 100644 --- a/src/os/linux/inc/os.h +++ b/src/os/linux/inc/os.h @@ -236,6 +236,9 @@ void taosSetCoreDump(); void taosBlockSIGPIPE(); +int tSystem(const char * cmd) ; + + #ifdef _ALPINE typedef int(*__compar_fn_t)(const void *, const void *); void error (int, int, const char *); diff --git a/src/os/linux/src/linuxPlatform.c b/src/os/linux/src/linuxPlatform.c index a8ecf43c46..9a38c98f81 100644 --- a/src/os/linux/src/linuxPlatform.c +++ b/src/os/linux/src/linuxPlatform.c @@ -241,3 +241,32 @@ void taosBlockSIGPIPE() { uError("failed to block SIGPIPE"); } } + +int tSystem(const char * cmd) +{ + FILE * fp; + int res; + char buf[1024]; + if (cmd == NULL) { + uError("tSystem cmd is NULL!\n"); + return -1; + } + + if ((fp = popen(cmd, "r") ) == NULL) { + uError("popen cmd:%s error: %s/n", cmd, strerror(errno)); + return -1; + } else { + while(fgets(buf, sizeof(buf), fp)) { + uDebug("popen result:%s", buf); + } + + if ((res = pclose(fp)) == -1) { + uError("close popen file pointer fp error!\n"); + } else { + uDebug("popen res is :%d\n", res); + } + + return res; + } +} + diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index 949e9eb109..1ba57bbaaa 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -184,6 +184,7 @@ uint32_t ip2uint(const char *const ip_addr); void taosRemoveDir(char *rootDir); int tmkdir(const char *pathname, mode_t mode); +void taosMvDir(char* destDir, char *srcDir); #define TAOS_ALLOC_MODE_DEFAULT 0 #define TAOS_ALLOC_MODE_RANDOM_FAIL 1 diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index cc720eb5f7..9c5bffef95 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -799,3 +799,13 @@ int tmkdir(const char *path, mode_t mode) { if (code < 0 && errno == EEXIST) code = 0; return code; } + +void taosMvDir(char* destDir, char *srcDir) { + char shellCmd[1024+1] = {0}; + + //(void)snprintf(shellCmd, 1024, "cp -rf %s %s", srcDir, destDir); + (void)snprintf(shellCmd, 1024, "mv %s %s", srcDir, destDir); + tSystem(shellCmd); + uInfo("shell cmd:%s is executed", shellCmd); +} + diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index cbd7953072..f71f6adefb 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -347,6 +347,7 @@ void vnodeRelease(void *pVnodeRaw) { if (pVnode->status == TAOS_VN_STATUS_DELETING) { char rootDir[TSDB_FILENAME_LEN] = {0}; sprintf(rootDir, "%s/vnode%d", tsVnodeDir, vgId); + taosMvDir(tsVnodeBakDir, rootDir); taosRemoveDir(rootDir); } From c9600717ad4dcb7bbaabe041506d4b978ab1516d Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 1 Jul 2020 15:58:59 +0800 Subject: [PATCH 14/31] fix compile error and reuse getNullValue --- src/common/src/ttypes.c | 7 ++++--- src/cq/src/cqMain.c | 27 +++++---------------------- 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index 8ea4fe906a..eff25e8c93 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -471,10 +471,11 @@ static uint32_t nullInt = TSDB_DATA_INT_NULL; static uint64_t nullBigInt = TSDB_DATA_BIGINT_NULL; static uint32_t nullFloat = TSDB_DATA_FLOAT_NULL; static uint64_t nullDouble = TSDB_DATA_DOUBLE_NULL; -static struct { + +static union { tstr str; - int32_t pad; -} nullBinary = {.str = {.len = 1}, .pad = 0}, nullNchar = {.str = {.len = 4}, .pad = 0}; + char pad[sizeof(tstr) + 4]; +} nullBinary = {.str = {.len = 1}}, nullNchar = {.str = {.len = 4}}; static void *nullValues[] = { &nullBool, &nullTinyInt, &nullSmallInt, &nullInt, &nullBigInt, diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 098d69fcb2..3e7e8525ef 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -256,30 +256,13 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { SDataRow trow = (SDataRow)pBlk->data; tdInitDataRow(trow, pSchema); - union { - char buf[sizeof(int64_t)]; - tstr str; - } nullVal; - for (int32_t i = 0; i < pSchema->numOfCols; i++) { STColumn *c = pSchema->columns + i; - char* val = (char*)row[i]; - if (IS_VAR_DATA_TYPE(c->type)) { - if (val == NULL) { - val = nullVal.buf; - if (c->type == TSDB_DATA_TYPE_BINARY) { - setNull(nullVal.str.data, TSDB_DATA_TYPE_BINARY, 1); - nullVal.str.len = 1; - } else { - setNull(nullVal.str.data, TSDB_DATA_TYPE_NCHAR, 4); - nullVal.str.len = 4; - } - } else { - val -= sizeof(VarDataLenT); - } - } else if (val == NULL) { - val = nullVal.buf; - setNull(val, c->type, c->bytes); + void* val = row[i]; + if (val == NULL) { + val = getNullValue(c->type); + } else if (IS_VAR_DATA_TYPE(c->type)) { + val = ((char*)val) - sizeof(VarDataLenT); } tdAppendColVal(trow, val, c->type, c->bytes, c->offset); } From 2ed7cdd35f7a7cf637c36ce94c50fef221dc2eaa Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 1 Jul 2020 16:57:11 +0800 Subject: [PATCH 15/31] remove debug printf --- src/tsdb/src/tsdbMeta.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 4f331d8131..dafc7dbb1b 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -853,7 +853,6 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { STable *pSTable = tsdbGetTableByUid(pMeta, TABLE_SUID(pTable)); ASSERT(pSTable != NULL); - printf("pTable = %p, pSTable = %p\n", pTable, pSTable); pTable->pSuper = pSTable; int32_t level = 0; From d616e0f1c8912b0569bb617fe0dfb19996579dcf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 1 Jul 2020 17:15:57 +0800 Subject: [PATCH 16/31] [td-822] --- src/client/src/tscServer.c | 16 ++-- src/client/src/tscSystem.c | 2 +- src/inc/query.h | 11 +-- src/mnode/src/mnodeProfile.c | 16 ++-- src/mnode/src/mnodeShow.c | 10 +- src/plugins/http/src/httpContext.c | 10 +- src/plugins/http/src/httpSession.c | 10 +- src/query/inc/qExecutor.h | 20 ++-- src/query/src/qExecutor.c | 38 ++++---- src/util/inc/tcache.h | 48 +++++----- src/util/src/tcache.c | 141 +++++++++++++++++------------ src/vnode/inc/vnodeInt.h | 1 + src/vnode/src/vnodeMain.c | 30 ++++-- src/vnode/src/vnodeRead.c | 74 +++++++++------ 14 files changed, 249 insertions(+), 178 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 8dd5ef69f7..eb3235d9e4 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1674,8 +1674,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); assert(pTableMetaInfo->pTableMeta == NULL); - pTableMetaInfo->pTableMeta = - (STableMeta *) taosCachePut(tscCacheHandle, pTableMetaInfo->name, pTableMeta, size, tsTableMetaKeepTimer); + pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscCacheHandle, pTableMetaInfo->name, + strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer); // todo handle out of memory case if (pTableMetaInfo->pTableMeta == NULL) { @@ -1878,7 +1878,8 @@ int tscProcessShowRsp(SSqlObj *pSql) { size_t size = 0; STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); - pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, (char *)pTableMeta, size, tsTableMetaKeepTimer); + pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, strlen(key), (char *)pTableMeta, size, + tsTableMetaKeepTimer); SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); if (pQueryInfo->colList == NULL) { @@ -1948,9 +1949,8 @@ int tscProcessDropDbRsp(SSqlObj *UNUSED_PARAM(pSql)) { int tscProcessDropTableRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pTableMeta = taosCacheAcquireByName(tscCacheHandle, pTableMetaInfo->name); - if (pTableMeta == NULL) { - /* not in cache, abort */ + STableMeta *pTableMeta = taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); + if (pTableMeta == NULL) { /* not in cache, abort */ return 0; } @@ -1974,7 +1974,7 @@ int tscProcessDropTableRsp(SSqlObj *pSql) { int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pTableMeta = taosCacheAcquireByName(tscCacheHandle, pTableMetaInfo->name); + STableMeta *pTableMeta = taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); if (pTableMeta == NULL) { /* not in cache, abort */ return 0; } @@ -2124,7 +2124,7 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), false); } - pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByName(tscCacheHandle, pTableMetaInfo->name); + pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); if (pTableMetaInfo->pTableMeta != NULL) { STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); tscDebug("%p retrieve table Meta from cache, the number of columns:%d, numOfTags:%d, %p", pSql, tinfo.numOfColumns, diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 201ace43de..f1d69fa261 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -148,7 +148,7 @@ void taos_init_imp() { refreshTime = refreshTime < 10 ? 10 : refreshTime; if (tscCacheHandle == NULL) { - tscCacheHandle = taosCacheInit(refreshTime); + tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL); } tscDebug("client is initialized successfully"); diff --git a/src/inc/query.h b/src/inc/query.h index 5fd2ede034..af3a89682c 100644 --- a/src/inc/query.h +++ b/src/inc/query.h @@ -20,6 +20,7 @@ extern "C" { #endif typedef void* qinfo_t; +typedef void (*_qinfo_free_fn_t)(void*); /** * create the qinfo object according to QueryTableMsg @@ -28,15 +29,13 @@ typedef void* qinfo_t; * @param qinfo * @return */ -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo); +int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, void* param, _qinfo_free_fn_t fn, qinfo_t* qinfo); /** * Destroy QInfo object * @param qinfo qhandle - * @param fp destroy callback function, while the qhandle is destoried, invoke the fp - * @param param free callback params */ -void qDestroyQueryInfo(qinfo_t qinfo, void (*fp)(void*), void* param); +void qDestroyQueryInfo(qinfo_t qinfo); /** * the main query execution function, including query on both table and multitables, @@ -81,11 +80,9 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo); /** * kill current ongoing query and free query handle automatically * @param qinfo qhandle - * @param fp destroy callback function, while the qhandle is destoried, invoke the fp - * @param param free callback params * @return */ -int32_t qKillQuery(qinfo_t qinfo, void (*fp)(void*), void* param); +int32_t qKillQuery(qinfo_t qinfo); #ifdef __cplusplus } diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index f3d6a3d344..a1d4be93c6 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -68,7 +68,7 @@ int32_t mnodeInitProfile() { mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_STREAM, mnodeProcessKillStreamMsg); mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_CONN, mnodeProcessKillConnectionMsg); - tsMnodeConnCache = taosCacheInitWithCb(CONN_CHECK_TIME, mnodeFreeConn); + tsMnodeConnCache = taosCacheInitWithCb(TSDB_DATA_TYPE_BINARY, CONN_CHECK_TIME,false, mnodeFreeConn); return 0; } @@ -101,8 +101,8 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) { tstrncpy(connObj.user, user, sizeof(connObj.user)); char key[10]; - sprintf(key, "%u", connId); - SConnObj *pConn = taosCachePut(tsMnodeConnCache, key, &connObj, sizeof(connObj), CONN_KEEP_TIME); + int32_t len = sprintf(key, "%u", connId); + SConnObj *pConn = taosCachePut(tsMnodeConnCache, key, len, &connObj, sizeof(connObj), CONN_KEEP_TIME); mDebug("connId:%d, is created, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port); return pConn; @@ -115,10 +115,10 @@ void mnodeReleaseConn(SConnObj *pConn) { SConnObj *mnodeAccquireConn(uint32_t connId, char *user, uint32_t ip, uint16_t port) { char key[10]; - sprintf(key, "%u", connId); + int32_t len = sprintf(key, "%u", connId); uint64_t expireTime = CONN_KEEP_TIME * 1000 + (uint64_t)taosGetTimestampMs(); - SConnObj *pConn = taosCacheUpdateExpireTimeByName(tsMnodeConnCache, key, expireTime); + SConnObj *pConn = taosCacheUpdateExpireTimeByName(tsMnodeConnCache, key, len, expireTime); if (pConn == NULL) { mError("connId:%d, is already destroyed, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port); return NULL; @@ -547,7 +547,7 @@ static int32_t mnodeProcessKillQueryMsg(SMnodeMsg *pMsg) { int32_t queryId = (int32_t)strtol(queryIdStr, NULL, 10); - SConnObj *pConn = taosCacheAcquireByName(tsMnodeConnCache, connIdStr); + SConnObj *pConn = taosCacheAcquireByKey(tsMnodeConnCache, connIdStr, strlen(connIdStr)); if (pConn == NULL) { mError("connId:%s, failed to kill queryId:%d, conn not exist", connIdStr, queryId); return TSDB_CODE_MND_INVALID_CONN_ID; @@ -577,7 +577,7 @@ static int32_t mnodeProcessKillStreamMsg(SMnodeMsg *pMsg) { int32_t streamId = (int32_t)strtol(streamIdStr, NULL, 10); - SConnObj *pConn = taosCacheAcquireByName(tsMnodeConnCache, connIdStr); + SConnObj *pConn = taosCacheAcquireByKey(tsMnodeConnCache, connIdStr, strlen(connIdStr)); if (pConn == NULL) { mError("connId:%s, failed to kill streamId:%d, conn not exist", connIdStr, streamId); return TSDB_CODE_MND_INVALID_CONN_ID; @@ -594,7 +594,7 @@ static int32_t mnodeProcessKillConnectionMsg(SMnodeMsg *pMsg) { if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS; SCMKillConnMsg *pKill = pMsg->rpcMsg.pCont; - SConnObj * pConn = taosCacheAcquireByName(tsMnodeConnCache, pKill->queryId); + SConnObj * pConn = taosCacheAcquireByKey(tsMnodeConnCache, pKill->queryId, strlen(pKill->queryId)); if (pConn == NULL) { mError("connId:%s, failed to kill, conn not exist", pKill->queryId); return TSDB_CODE_MND_INVALID_CONN_ID; diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 6f5216dfa3..8999ce4083 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -65,7 +65,7 @@ int32_t mnodeInitShow() { mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mnodeProcessConnectMsg); mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_USE_DB, mnodeProcessUseMsg); - tsMnodeShowCache = taosCacheInitWithCb(5, mnodeFreeShowObj); + tsMnodeShowCache = taosCacheInitWithCb(TSDB_DATA_TYPE_BINARY, 5, false, mnodeFreeShowObj); return 0; } @@ -364,9 +364,9 @@ static bool mnodeCheckShowFinished(SShowObj *pShow) { static bool mnodeAccquireShowObj(SShowObj *pShow) { char key[10]; - sprintf(key, "%d", pShow->index); + int32_t len = sprintf(key, "%d", pShow->index); - SShowObj *pSaved = taosCacheAcquireByName(tsMnodeShowCache, key); + SShowObj *pSaved = taosCacheAcquireByKey(tsMnodeShowCache, key, len); if (pSaved == pShow) { mDebug("%p, show is accquired from cache", pShow); return true; @@ -379,9 +379,9 @@ static void *mnodePutShowObj(SShowObj *pShow, int32_t size) { if (tsMnodeShowCache != NULL) { char key[10]; pShow->index = atomic_add_fetch_32(&tsShowObjIndex, 1); - sprintf(key, "%d", pShow->index); + int32_t len = sprintf(key, "%d", pShow->index); - SShowObj *newQhandle = taosCachePut(tsMnodeShowCache, key, pShow, size, 6); + SShowObj *newQhandle = taosCachePut(tsMnodeShowCache, key, len, pShow, size, 6); free(pShow); mDebug("%p, show is put into cache, index:%s", newQhandle, key); diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index b078be5930..7fb9720170 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -58,7 +58,7 @@ static void httpDestroyContext(void *data) { } bool httpInitContexts() { - tsHttpServer.contextCache = taosCacheInitWithCb(2, httpDestroyContext); + tsHttpServer.contextCache = taosCacheInitWithCb(TSDB_DATA_TYPE_BINARY, 2, false, httpDestroyContext); if (tsHttpServer.contextCache == NULL) { httpError("failed to init context cache"); return false; @@ -104,14 +104,14 @@ HttpContext *httpCreateContext(int32_t fd) { if (pContext == NULL) return NULL; char contextStr[16] = {0}; - snprintf(contextStr, sizeof(contextStr), "%p", pContext); + int32_t keySize = snprintf(contextStr, sizeof(contextStr), "%p", pContext); pContext->fd = fd; pContext->httpVersion = HTTP_VERSION_10; pContext->lastAccessTime = taosGetTimestampSec(); pContext->state = HTTP_CONTEXT_STATE_READY; - HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, contextStr, &pContext, sizeof(HttpContext *), 3); + HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, contextStr, keySize, &pContext, sizeof(HttpContext *), 3); pContext->ppContext = ppContext; httpDebug("context:%p, fd:%d, is created, item:%p", pContext, fd, ppContext); @@ -123,9 +123,9 @@ HttpContext *httpCreateContext(int32_t fd) { HttpContext *httpGetContext(void *ptr) { char contextStr[16] = {0}; - snprintf(contextStr, sizeof(contextStr), "%p", ptr); + int32_t len = snprintf(contextStr, sizeof(contextStr), "%p", ptr); - HttpContext **ppContext = taosCacheAcquireByName(tsHttpServer.contextCache, contextStr); + HttpContext **ppContext = taosCacheAcquireByKey(tsHttpServer.contextCache, contextStr, len); if (ppContext) { HttpContext *pContext = *ppContext; diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c index 83602e1291..3a901167d5 100644 --- a/src/plugins/http/src/httpSession.c +++ b/src/plugins/http/src/httpSession.c @@ -33,9 +33,9 @@ void httpCreateSession(HttpContext *pContext, void *taos) { memset(&session, 0, sizeof(HttpSession)); session.taos = taos; session.refCount = 1; - snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); + int32_t len = snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); - pContext->session = taosCachePut(server->sessionCache, session.id, &session, sizeof(HttpSession), tsHttpSessionExpire); + pContext->session = taosCachePut(server->sessionCache, session.id, len, &session, sizeof(HttpSession), tsHttpSessionExpire); // void *temp = pContext->session; // taosCacheRelease(server->sessionCache, (void **)&temp, false); @@ -57,9 +57,9 @@ static void httpFetchSessionImp(HttpContext *pContext) { pthread_mutex_lock(&server->serverMutex); char sessionId[HTTP_SESSION_ID_LEN]; - snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); + int32_t len = snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); - pContext->session = taosCacheAcquireByName(server->sessionCache, sessionId); + pContext->session = taosCacheAcquireByKey(server->sessionCache, sessionId, len); if (pContext->session != NULL) { atomic_add_fetch_32(&pContext->session->refCount, 1); httpDebug("context:%p, fd:%d, ip:%s, user:%s, find an exist session:%p:%p, sessionRef:%d", pContext, pContext->fd, @@ -115,7 +115,7 @@ void httpCleanUpSessions() { } bool httpInitSessions() { - tsHttpServer.sessionCache = taosCacheInitWithCb(5, httpDestroySession); + tsHttpServer.sessionCache = taosCacheInitWithCb(TSDB_DATA_TYPE_BINARY, 5, false, httpDestroySession); if (tsHttpServer.sessionCache == NULL) { httpError("failed to init session cache"); return false; diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index a830723845..d7d59230b6 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -27,6 +27,7 @@ #include "tref.h" #include "tsdb.h" #include "tsqlfunction.h" +#include "query.h" struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); @@ -181,13 +182,13 @@ typedef struct SQueryRuntimeEnv { } SQueryRuntimeEnv; typedef struct SQInfo { - void* signature; - int32_t pointsInterpo; - int32_t code; // error code to returned to client - sem_t dataReady; - void* tsdb; - int32_t vgId; - + void* signature; + int32_t pointsInterpo; + int32_t code; // error code to returned to client + sem_t dataReady; + void* tsdb; + void* param; + int32_t vgId; STableGroupInfo tableGroupInfo; // table id list < only includes the STable list> STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure SQueryRuntimeEnv runtimeEnv; @@ -202,8 +203,9 @@ typedef struct SQInfo { * We later may refactor to remove this attribution by using another flag to denote * whether a multimeter query is completed or not. */ - int32_t tableIndex; - int32_t numOfGroupResultPages; + int32_t tableIndex; + int32_t numOfGroupResultPages; + _qinfo_free_fn_t fn; } SQInfo; #endif // TDENGINE_QUERYEXECUTOR_H diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 9df161b3dc..5f5242cefd 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4069,7 +4069,7 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) { return pFillCol; } -int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool isSTableQuery) { +int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bool isSTableQuery, void* freeParam, _qinfo_free_fn_t fn) { int32_t code = TSDB_CODE_SUCCESS; SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; @@ -4083,14 +4083,16 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool pQInfo->tsdb = tsdb; pQInfo->vgId = vgId; + pQInfo->param = freeParam; + pQInfo->fn = fn; pRuntimeEnv->pQuery = pQuery; - pRuntimeEnv->pTSBuf = param; + pRuntimeEnv->pTSBuf = pTsBuf; pRuntimeEnv->cur.vgroupIndex = -1; pRuntimeEnv->stableQuery = isSTableQuery; pRuntimeEnv->prevGroupId = INT32_MIN; - if (param != NULL) { + if (pTsBuf != NULL) { int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; tsBufSetTraverseOrder(pRuntimeEnv->pTSBuf, order); } @@ -5697,8 +5699,7 @@ static bool isValidQInfo(void *param) { return (sig == (uint64_t)pQInfo); } - -static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable) { +static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable, void* param, _qinfo_free_fn_t fn) { int32_t code = TSDB_CODE_SUCCESS; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -5731,7 +5732,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ } // filter the qualified - if ((code = doInitQInfo(pQInfo, pTSBuf, tsdb, vgId, isSTable)) != TSDB_CODE_SUCCESS) { + if ((code = doInitQInfo(pQInfo, pTSBuf, tsdb, vgId, isSTable, param, fn)) != TSDB_CODE_SUCCESS) { goto _error; } @@ -5894,7 +5895,8 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { return TSDB_CODE_SUCCESS; } -int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) { +int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, void* param, _qinfo_free_fn_t fn, + qinfo_t* pQInfo) { assert(pQueryMsg != NULL); int32_t code = TSDB_CODE_SUCCESS; @@ -5984,7 +5986,7 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi goto _over; } - code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery); + code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery, param, fn); _over: free(tagCond); @@ -6020,7 +6022,7 @@ static void doDestoryQueryInfo(SQInfo* pQInfo) { freeQInfo(pQInfo); } -void qDestroyQueryInfo(qinfo_t qHandle, void (*fp)(void*), void* param) { +void qDestroyQueryInfo(qinfo_t qHandle) { SQInfo* pQInfo = (SQInfo*) qHandle; if (!isValidQInfo(pQInfo)) { return; @@ -6030,11 +6032,12 @@ void qDestroyQueryInfo(qinfo_t qHandle, void (*fp)(void*), void* param) { qDebug("QInfo:%p dec refCount, value:%d", pQInfo, ref); if (ref == 0) { - doDestoryQueryInfo(pQInfo); - - if (fp != NULL) { - fp(param); + if (pQInfo->fn != NULL) { + assert(pQInfo->param != NULL); + pQInfo->fn(pQInfo->param); } + + doDestoryQueryInfo(pQInfo); } } @@ -6048,7 +6051,7 @@ void qTableQuery(qinfo_t qinfo, void (*fp)(void*), void* param) { if (isQueryKilled(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); - qDestroyQueryInfo(pQInfo, fp, param); + qDestroyQueryInfo(pQInfo); return; } @@ -6069,7 +6072,7 @@ void qTableQuery(qinfo_t qinfo, void (*fp)(void*), void* param) { } sem_post(&pQInfo->dataReady); - qDestroyQueryInfo(pQInfo, fp, param); + qDestroyQueryInfo(pQInfo); } int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) { @@ -6162,7 +6165,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co return code; } -int32_t qKillQuery(qinfo_t qinfo, void (*fp)(void*), void* param) { +int32_t qKillQuery(qinfo_t qinfo) { SQInfo *pQInfo = (SQInfo *)qinfo; if (pQInfo == NULL || !isValidQInfo(pQInfo)) { @@ -6170,8 +6173,7 @@ int32_t qKillQuery(qinfo_t qinfo, void (*fp)(void*), void* param) { } setQueryKilled(pQInfo); - qDestroyQueryInfo(pQInfo, fp, param); - + qDestroyQueryInfo(pQInfo); return TSDB_CODE_SUCCESS; } diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index 17b3823831..cd3d0d436f 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -24,6 +24,8 @@ extern "C" { #include "tref.h" #include "hash.h" +typedef void (*__cache_freeres_fn_t)(void*); + typedef struct SCacheStatis { int64_t missCount; int64_t hitCount; @@ -34,14 +36,15 @@ typedef struct SCacheStatis { typedef struct SCacheDataNode { uint64_t addedTime; // the added time when this element is added or updated into cache - uint64_t expiredTime; // expiredTime expiredTime when this element should be remove from cache + uint64_t lifespan; // expiredTime expiredTime when this element should be remove from cache uint64_t signature; uint32_t size; // allocated size for current SCacheDataNode - uint16_t keySize: 15; - bool inTrashCan: 1;// denote if it is in trash or not T_REF_DECLARE() - char *key; - char data[]; + uint16_t keySize: 15; // max key size: 32kb + bool inTrashCan: 1;// denote if it is in trash or not + int32_t extendFactor; // number of life span extend + char *key; + char data[]; } SCacheDataNode; typedef struct STrashElem { @@ -62,29 +65,32 @@ typedef struct { int64_t totalSize; // total allocated buffer in this hash table, SCacheObj is not included. int64_t refreshTime; STrashElem * pTrash; - void * tmrCtrl; - void * pTimer; +// void * tmrCtrl; +// void * pTimer; SCacheStatis statistics; SHashObj * pHashTable; - _hash_free_fn_t freeFp; + __cache_freeres_fn_t freeFp; uint32_t numOfElemsInTrash; // number of element in trash uint8_t deleting; // set the deleting flag to stop refreshing ASAP. pthread_t refreshWorker; - + bool extendLifespan; // auto extend life span when one item is accessed. #if defined(LINUX) pthread_rwlock_t lock; #else - pthread_mutex_t lock; + pthread_mutex_t lock; #endif } SCacheObj; /** * initialize the cache object - * @param refreshTime refresh operation interval time, the maximum survival time when one element is expired and - * not referenced by other objects + * @param keyType key type + * @param refreshTimeInSeconds refresh operation interval time, the maximum survival time when one element is expired + * and not referenced by other objects + * @param extendLifespan auto extend lifespan, if accessed + * @param fn free resource callback function * @return */ -SCacheObj *taosCacheInit(int64_t refreshTimeInSeconds); +SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_freeres_fn_t fn); /** * initialize the cache object and set the free object callback function @@ -92,7 +98,7 @@ SCacheObj *taosCacheInit(int64_t refreshTimeInSeconds); * @param freeCb * @return */ -SCacheObj *taosCacheInitWithCb(int64_t refreshTimeInSeconds, void (*freeCb)(void *data)); +SCacheObj *taosCacheInitWithCb(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_freeres_fn_t fn); /** * add data into cache @@ -104,7 +110,7 @@ SCacheObj *taosCacheInitWithCb(int64_t refreshTimeInSeconds, void (*freeCb)(void * @param keepTime survival time in second * @return cached element */ -void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, size_t dataSize, int keepTimeInSeconds); +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int keepTimeInSeconds); /** * get data from cache @@ -112,22 +118,23 @@ void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, siz * @param key key * @return cached data or NULL */ -void *taosCacheAcquireByName(SCacheObj *pCacheObj, const char *key); +void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen); /** * update the expire time of data in cache * @param pCacheObj cache object * @param key key + * @param keyLen keyLen * @param expireTime new expire time of data * @return */ -void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, const char *key, uint64_t expireTime); +void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, const char *key, size_t keyLen, uint64_t expireTime); /** * Add one reference count for the exist data, and assign this data for a new owner. * The new owner needs to invoke the taosCacheRelease when it does not need this data anymore. - * This procedure is a faster version of taosCacheAcquireByName function, which avoids the sideeffect of the problem of - * the data is moved to trash, and taosCacheAcquireByName will fail to retrieve it again. + * This procedure is a faster version of taosCacheAcquireByKey function, which avoids the sideeffect of the problem of + * the data is moved to trash, and taosCacheAcquireByKey will fail to retrieve it again. * * @param handle * @param data @@ -148,8 +155,7 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data); * if it is referenced by other object, it will be remain in cache * @param handle cache object * @param data not the key, actually referenced data - * @param _remove force model, reduce the ref count and move the data into - * pTrash + * @param _remove force model, reduce the ref count and move the data into pTrash */ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove); diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 2dd641731c..91d559a92b 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -167,7 +167,7 @@ static SCacheDataNode *taosUpdateCacheImpl(SCacheObj *pCacheObj, SCacheDataNode // update the timestamp information for updated key/value pNewNode->addedTime = taosGetTimestampMs(); - pNewNode->expiredTime = pNewNode->addedTime + duration; + pNewNode->lifespan = duration; T_REF_INC(pNewNode); @@ -224,8 +224,8 @@ static void doCleanupDataCache(SCacheObj *pCacheObj); */ static void* taosCacheRefresh(void *handle); -SCacheObj *taosCacheInitWithCb(int64_t refreshTime, void (*freeCb)(void *data)) { - if (refreshTime <= 0) { +SCacheObj *taosCacheInitWithCb(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_freeres_fn_t fn) { + if (refreshTimeInSeconds <= 0) { return NULL; } @@ -235,7 +235,7 @@ SCacheObj *taosCacheInitWithCb(int64_t refreshTime, void (*freeCb)(void *data)) return NULL; } - pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false); + pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(keyType), false); if (pCacheObj->pHashTable == NULL) { free(pCacheObj); uError("failed to allocate memory, reason:%s", strerror(errno)); @@ -243,10 +243,9 @@ SCacheObj *taosCacheInitWithCb(int64_t refreshTime, void (*freeCb)(void *data)) } // set free cache node callback function for hash table - // taosHashSetFreecb(pCacheObj->pHashTable, taosFreeNode); - - pCacheObj->freeFp = freeCb; - pCacheObj->refreshTime = refreshTime * 1000; + pCacheObj->freeFp = fn; + pCacheObj->refreshTime = refreshTimeInSeconds * 1000; + pCacheObj->extendLifespan = extendLifespan; if (__cache_lock_init(pCacheObj) != 0) { taosHashCleanup(pCacheObj->pHashTable); @@ -256,7 +255,7 @@ SCacheObj *taosCacheInitWithCb(int64_t refreshTime, void (*freeCb)(void *data)) return NULL; } - pthread_attr_t thattr; + pthread_attr_t thattr = {0}; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); @@ -266,19 +265,17 @@ SCacheObj *taosCacheInitWithCb(int64_t refreshTime, void (*freeCb)(void *data)) return pCacheObj; } -SCacheObj *taosCacheInit(int64_t refreshTime) { - return taosCacheInitWithCb(refreshTime, NULL); +SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_freeres_fn_t fn) { + return taosCacheInitWithCb(keyType, refreshTimeInSeconds, extendLifespan, fn); } -void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, size_t dataSize, int duration) { +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int duration) { SCacheDataNode *pNode; if (pCacheObj == NULL || pCacheObj->pHashTable == NULL) { return NULL; } - size_t keyLen = strlen(key); - __cache_wr_lock(pCacheObj); SCacheDataNode **pt = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); SCacheDataNode * pOld = (pt != NULL) ? (*pt) : NULL; @@ -288,14 +285,14 @@ void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, siz if (NULL != pNode) { pCacheObj->totalSize += pNode->size; - uDebug("key:%s %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", total:%" PRId64 ", size:%" PRId64 " bytes", - key, pNode, pNode->addedTime, pNode->expiredTime, pCacheObj->totalSize, dataSize); + uDebug("key:%p %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", total:%" PRId64 ", size:%" PRId64 " bytes", + key, pNode, pNode->addedTime, (pNode->lifespan * pNode->extendFactor + pNode->addedTime), pCacheObj->totalSize, dataSize); } else { - uError("key:%s failed to added into cache, out of memory", key); + uError("key:%p failed to added into cache, out of memory", key); } } else { // old data exists, update the node pNode = taosUpdateCacheImpl(pCacheObj, pOld, key, keyLen, pData, dataSize, duration * 1000L); - uDebug("key:%s %p exist in cache, updated", key, pNode); + uDebug("key:%p %p exist in cache, updated", key, pNode); } __cache_unlock(pCacheObj); @@ -303,47 +300,55 @@ void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, siz return (pNode != NULL) ? pNode->data : NULL; } -void *taosCacheAcquireByName(SCacheObj *pCacheObj, const char *key) { +void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen) { if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) == 0) { return NULL; } - uint32_t keyLen = (uint32_t)strlen(key); - __cache_rd_lock(pCacheObj); SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); + + int32_t ref = 0; if (ptNode != NULL) { - T_REF_INC(*ptNode); + ref = T_REF_INC(*ptNode); + + // if the remained life span is less then the (*ptNode)->lifeSpan, add up one lifespan + if (pCacheObj->extendLifespan) { + int64_t now = taosGetTimestampMs(); + + if ((now - (*ptNode)->addedTime) < (*ptNode)->lifespan * (*ptNode)->extendFactor) { + (*ptNode)->extendFactor += 1; + uDebug("key:%p extend life time to %"PRId64, key, (*ptNode)->lifespan * (*ptNode)->extendFactor + (*ptNode)->addedTime); + } + } } - __cache_unlock(pCacheObj); if (ptNode != NULL) { atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); - uDebug("key:%s is retrieved from cache, %p refcnt:%d", key, (*ptNode), T_REF_VAL_GET(*ptNode)); + uDebug("key:%p is retrieved from cache, %p refcnt:%d", key, (*ptNode), ref); } else { atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); - uDebug("key:%s not in cache, retrieved failed", key); + uDebug("key:%p not in cache, retrieved failed", key); } atomic_add_fetch_32(&pCacheObj->statistics.totalAccess, 1); return (ptNode != NULL) ? (*ptNode)->data : NULL; } -void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, const char *key, uint64_t expireTime) { +void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, const char *key, size_t keyLen, uint64_t expireTime) { if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) == 0) { return NULL; } - - uint32_t keyLen = (uint32_t)strlen(key); - + __cache_rd_lock(pCacheObj); SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); if (ptNode != NULL) { T_REF_INC(*ptNode); - (*ptNode)->expiredTime = expireTime; + (*ptNode)->extendFactor += 1; +// (*ptNode)->lifespan = expireTime; } __cache_unlock(pCacheObj); @@ -373,7 +378,17 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { int32_t ref = T_REF_INC(ptNode); uDebug("%p acquired by data in cache, refcnt:%d", ptNode, ref) - + + // if the remained life span is less then the (*ptNode)->lifeSpan, add up one lifespan + if (pCacheObj->extendLifespan) { + int64_t now = taosGetTimestampMs(); + + if ((now - ptNode->addedTime) < ptNode->lifespan * ptNode->extendFactor) { + ptNode->extendFactor += 1; + uDebug("key:%p extend life time to %"PRId64, ptNode, ptNode->lifespan * ptNode->extendFactor + ptNode->addedTime); + } + } + // the data if referenced by at least one object, so the reference count must be greater than the value of 2. assert(ref >= 2); return data; @@ -408,7 +423,6 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { size_t offset = offsetof(SCacheDataNode, data); SCacheDataNode *pNode = (SCacheDataNode *)((char *)(*data) - offset); - if (pNode->signature != (uint64_t)pNode) { uError("key: %p release invalid cache data", pNode); return; @@ -420,9 +434,16 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { if (_remove) { __cache_wr_lock(pCacheObj); - // pNode may be released immediately by other thread after the reference count of pNode is set to 0, - // So we need to lock it in the first place. - taosCacheMoveToTrash(pCacheObj, pNode); + + if (T_REF_VAL_GET(pNode) == 0) { + // remove directly, if not referenced by other users + taosCacheReleaseNode(pCacheObj, pNode); + } else { + // pNode may be released immediately by other thread after the reference count of pNode is set to 0, + // So we need to lock it in the first place. + taosCacheMoveToTrash(pCacheObj, pNode); + } + __cache_unlock(pCacheObj); } } @@ -473,11 +494,11 @@ SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char * memcpy(pNewNode->key, key, keyLen); - pNewNode->addedTime = (uint64_t)taosGetTimestampMs(); - pNewNode->expiredTime = pNewNode->addedTime + duration; - - pNewNode->signature = (uint64_t)pNewNode; - pNewNode->size = (uint32_t)totalSize; + pNewNode->addedTime = (uint64_t)taosGetTimestampMs(); + pNewNode->lifespan = duration; + pNewNode->extendFactor = 1; + pNewNode->signature = (uint64_t)pNewNode; + pNewNode->size = (uint32_t)totalSize; return pNewNode; } @@ -501,7 +522,7 @@ void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { pNode->inTrashCan = true; pCacheObj->numOfElemsInTrash++; - uDebug("key:%s %p move to trash, numOfElem in trash:%d", pNode->key, pNode, pCacheObj->numOfElemsInTrash); + uDebug("key:%p %p move to trash, numOfElem in trash:%d", pNode->key, pNode, pCacheObj->numOfElemsInTrash); } void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem) { @@ -522,7 +543,10 @@ void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem) { } pElem->pData->signature = 0; - if (pCacheObj->freeFp) pCacheObj->freeFp(pElem->pData->data); + if (pCacheObj->freeFp) { + pCacheObj->freeFp(pElem->pData->data); + } + free(pElem->pData); free(pElem); } @@ -549,7 +573,7 @@ void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) { } if (force || (T_REF_VAL_GET(pElem->pData) == 0)) { - uDebug("key:%s %p removed from trash. numOfElem in trash:%d", pElem->pData->key, pElem->pData, + uDebug("key:%p %p removed from trash. numOfElem in trash:%d", pElem->pData->key, pElem->pData, pCacheObj->numOfElemsInTrash - 1); STrashElem *p = pElem; @@ -569,9 +593,7 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable); while (taosHashIterNext(pIter)) { SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); - // if (pNode->expiredTime <= expiredTime && T_REF_VAL_GET(pNode) <= 0) { taosCacheReleaseNode(pCacheObj, pNode); - //} } taosHashDestroyIter(pIter); @@ -610,27 +632,32 @@ void* taosCacheRefresh(void *handle) { // reset the count value count = 0; - size_t num = taosHashGetSize(pCacheObj->pHashTable); - if (num == 0) { + size_t elemInHash = taosHashGetSize(pCacheObj->pHashTable); + if (elemInHash + pCacheObj->numOfElemsInTrash == 0) { continue; } - uint64_t expiredTime = taosGetTimestampMs(); pCacheObj->statistics.refreshCount++; - SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable); + // refresh data in hash table + if (elemInHash > 0) { + int64_t expiredTime = taosGetTimestampMs(); - __cache_wr_lock(pCacheObj); - while (taosHashIterNext(pIter)) { - SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); - if (pNode->expiredTime <= expiredTime && T_REF_VAL_GET(pNode) <= 0) { - taosCacheReleaseNode(pCacheObj, pNode); + SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable); + + __cache_wr_lock(pCacheObj); + while (taosHashIterNext(pIter)) { + SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); + if ((pNode->addedTime + pNode->lifespan * pNode->extendFactor) <= expiredTime && T_REF_VAL_GET(pNode) <= 0) { + taosCacheReleaseNode(pCacheObj, pNode); + } } + + __cache_unlock(pCacheObj); + + taosHashDestroyIter(pIter); } - __cache_unlock(pCacheObj); - - taosHashDestroyIter(pIter); taosTrashCanEmpty(pCacheObj, false); } diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index e428eae688..76e53f3962 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -53,6 +53,7 @@ typedef struct { STsdbCfg tsdbCfg; SSyncCfg syncCfg; SWalCfg walCfg; + void *qHandlePool; // query handle pool char *rootDir; char db[TSDB_DB_NAME_LEN]; } SVnodeObj; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 4693dc02d3..49bc35e913 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -15,19 +15,22 @@ #define _DEFAULT_SOURCE #include "os.h" + +#include "tcache.h" +#include "cJSON.h" +#include "dnode.h" #include "hash.h" #include "taoserror.h" #include "taosmsg.h" -#include "tutil.h" +#include "tglobal.h" #include "trpc.h" #include "tsdb.h" #include "ttime.h" #include "ttimer.h" -#include "cJSON.h" -#include "tglobal.h" -#include "dnode.h" +#include "tutil.h" #include "vnode.h" #include "vnodeInt.h" +#include "query.h" #define TSDB_VNODE_VERSION_CONTENT_LEN 31 @@ -43,6 +46,7 @@ static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uin static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index); static void vnodeNotifyRole(void *ahandle, int8_t role); static void vnodeNotifyFileSynced(void *ahandle, uint64_t fversion); +static void vnodeFreeqHandle(void* phandle); static pthread_once_t vnodeModuleInit = PTHREAD_ONCE_INIT; @@ -280,6 +284,9 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { if (pVnode->role == TAOS_SYNC_ROLE_MASTER) cqStart(pVnode->cq); + const int32_t REFRESH_HANDLE_INTERVAL = 2; // every 2 seconds, rfresh handle pool + pVnode->qHandlePool = taosCacheInit(TSDB_DATA_TYPE_BIGINT, REFRESH_HANDLE_INTERVAL, true, vnodeFreeqHandle); + pVnode->events = NULL; pVnode->status = TAOS_VN_STATUS_READY; vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); @@ -848,12 +855,12 @@ static int32_t vnodeReadVersion(SVnodeObj *pVnode) { goto PARSE_OVER; } - cJSON *version = cJSON_GetObjectItem(root, "version"); - if (!version || version->type != cJSON_Number) { + cJSON *ver = cJSON_GetObjectItem(root, "version"); + if (!ver || ver->type != cJSON_Number) { vError("vgId:%d, failed to read vnode version, version not found", pVnode->vgId); goto PARSE_OVER; } - pVnode->version = version->valueint; + pVnode->version = ver->valueint; terrno = TSDB_CODE_SUCCESS; vInfo("vgId:%d, read vnode version successfully, version:%" PRId64, pVnode->vgId, pVnode->version); @@ -864,3 +871,12 @@ PARSE_OVER: if(fp) fclose(fp); return terrno; } + +void vnodeFreeqHandle(void *qHandle) { + void** handle = qHandle; + if (handle == NULL || *handle == NULL) { + return; + } + + qKillQuery(*handle); +} \ No newline at end of file diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 0c08c77e32..52adffff7e 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -15,20 +15,21 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taosmsg.h" + +#include "tglobal.h" #include "taoserror.h" -#include "tqueue.h" +#include "taosmsg.h" +#include "tcache.h" +#include "query.h" #include "trpc.h" #include "tsdb.h" -#include "twal.h" -#include "tdataformat.h" #include "vnode.h" #include "vnodeInt.h" -#include "query.h" static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SReadMsg *pReadMsg); static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg); static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg); +static int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId); void vnodeInitReadFp(void) { vnodeProcessReadMsgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessQueryMsg; @@ -58,19 +59,6 @@ int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) { return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg); } -// notify connection(handle) that current qhandle is created, if current connection from -// client is broken, the query needs to be killed immediately. -static int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId) { - SRetrieveTableMsg* killQueryMsg = rpcMallocCont(sizeof(SRetrieveTableMsg)); - killQueryMsg->qhandle = htobe64((uint64_t) qhandle); - killQueryMsg->free = htons(1); - killQueryMsg->header.vgId = htonl(vgId); - killQueryMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); - - vDebug("QInfo:%p register qhandle to connect:%p", qhandle, handle); - return rpcReportProgress(handle, (char*) killQueryMsg, sizeof(SRetrieveTableMsg)); -} - static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { void * pCont = pReadMsg->pCont; int32_t contLen = pReadMsg->contLen; @@ -88,18 +76,25 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { vWarn("QInfo:%p connection %p broken, kill query", (void*)killQueryMsg->qhandle, pReadMsg->rpcMsg.handle); assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1); - // this message arrived here by means of the query message, so release the vnode is necessary - qKillQuery((qinfo_t) killQueryMsg->qhandle, vnodeRelease, pVnode); - vnodeRelease(pVnode); + // this message arrived here by means of the *query* message, so release the vnode is necessary + void** qhandle = taosCacheAcquireByKey(pVnode->qHandlePool, (void*) &killQueryMsg->qhandle, sizeof(killQueryMsg->qhandle)); + if (qhandle == NULL || *qhandle == NULL) { // todo handle invalid qhandle error + } else { +// qKillQuery((qinfo_t) killQueryMsg->qhandle); + taosCacheRelease(pVnode->qHandlePool, (void**) &qhandle, true); + } + + vnodeRelease(pVnode); return TSDB_CODE_TSC_QUERY_CANCELLED; // todo change the error code } int32_t code = TSDB_CODE_SUCCESS; qinfo_t pQInfo = NULL; + void** handle = NULL; if (contLen != 0) { - code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo); + code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, pVnode, vnodeRelease, &pQInfo); SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); pRsp->qhandle = htobe64((uint64_t) (pQInfo)); @@ -116,13 +111,15 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; // NOTE: there two refcount, needs to kill twice, todo refactor - qKillQuery(pQInfo, vnodeRelease, pVnode); - qKillQuery(pQInfo, vnodeRelease, pVnode); + // query has not been put into qhandle pool, kill it directly. + qKillQuery(pQInfo); + qKillQuery(pQInfo); return pRsp->code; } - vTrace("vgId:%d, QInfo:%p, dnode query msg disposed", pVnode->vgId, pQInfo); + handle = taosCachePut(pVnode->qHandlePool, pQInfo, sizeof(pQInfo), &pQInfo, sizeof(pQInfo), tsShellActivityTimer * 2); + assert(*handle == pQInfo); } else { assert(pQInfo == NULL); vnodeRelease(pVnode); @@ -138,6 +135,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (pQInfo != NULL) { qTableQuery(pQInfo, vnodeRelease, pVnode); // do execute query + taosCacheRelease(pVnode->qHandlePool, (void**) &handle, false); } return code; @@ -152,10 +150,18 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { pRetrieve->free = htons(pRetrieve->free); memset(pRet, 0, sizeof(SRspRet)); + int32_t ret = 0; + + void** handle = taosCacheAcquireByKey(pVnode->qHandlePool, &pQInfo, sizeof(pQInfo)); + if (handle == NULL || *handle != pQInfo) { + ret = TSDB_CODE_QRY_INVALID_QHANDLE; + } if (pRetrieve->free == 1) { vDebug("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, pQInfo); - int32_t ret = qKillQuery(pQInfo, vnodeRelease, pVnode); + + taosCacheRelease(pVnode->qHandlePool, handle, true); +// int32_t ret = qKillQuery(pQInfo); pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); pRet->len = sizeof(SRetrieveTableRsp); @@ -184,10 +190,24 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { pRet->qhandle = pQInfo; code = TSDB_CODE_VND_ACTION_NEED_REPROCESSED; } else { // no further execution invoked, release the ref to vnode - qDestroyQueryInfo(pQInfo, vnodeRelease, pVnode); + taosCacheRelease(pVnode->qHandlePool, (void**) &handle, true); +// qDestroyQueryInfo(pQInfo); } } vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed", pVnode->vgId, pQInfo); return code; } + +// notify connection(handle) that current qhandle is created, if current connection from +// client is broken, the query needs to be killed immediately. +int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId) { + SRetrieveTableMsg* killQueryMsg = rpcMallocCont(sizeof(SRetrieveTableMsg)); + killQueryMsg->qhandle = htobe64((uint64_t) qhandle); + killQueryMsg->free = htons(1); + killQueryMsg->header.vgId = htonl(vgId); + killQueryMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); + + vDebug("QInfo:%p register qhandle to connect:%p", qhandle, handle); + return rpcReportProgress(handle, (char*) killQueryMsg, sizeof(SRetrieveTableMsg)); +} \ No newline at end of file From f8f60c45bd85bdc8c9414481f763c9c7f533a8a9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 1 Jul 2020 17:19:14 +0800 Subject: [PATCH 17/31] [td-822] update the test script. --- src/util/tests/cacheTest.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/util/tests/cacheTest.cpp b/src/util/tests/cacheTest.cpp index 5762d5700b..43ac689ff4 100644 --- a/src/util/tests/cacheTest.cpp +++ b/src/util/tests/cacheTest.cpp @@ -19,12 +19,12 @@ int32_t tsMaxMeterConnections = 200; // test cache TEST(testCase, client_cache_test) { const int32_t REFRESH_TIME_IN_SEC = 2; - SCacheObj* tscCacheHandle = taosCacheInit(REFRESH_TIME_IN_SEC); + SCacheObj* tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, 0, NULL); const char* key1 = "test1"; char data1[] = "test11"; - char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data1, strlen(data1)+1, 1); + char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, strlen(key1), data1, strlen(data1)+1, 1); sleep(REFRESH_TIME_IN_SEC+1); printf("obj is still valid: %s\n", cachedObj); @@ -33,7 +33,7 @@ TEST(testCase, client_cache_test) { taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false); /* the object is cleared by cache clean operation */ - cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data2, strlen(data2)+1, 20); + cachedObj = (char*) taosCachePut(tscCacheHandle, key1, strlen(key1), data2, strlen(data2)+1, 20); printf("after updated: %s\n", cachedObj); printf("start to remove data from cache\n"); @@ -43,32 +43,32 @@ TEST(testCase, client_cache_test) { const char* key3 = "test2"; const char* data3 = "kkkkkkk"; - char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, data3, strlen(data3) + 1, 1); + char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, strlen(key3), data3, strlen(data3) + 1, 1); printf("%s\n", cachedObj2); taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); sleep(3); - char* d = (char*) taosCacheAcquireByName(tscCacheHandle, key3); + char* d = (char*) taosCacheAcquireByKey(tscCacheHandle, key3, strlen(key3)); // assert(d == NULL); char key5[] = "test5"; char data5[] = "data5kkkkk"; - cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data5, strlen(data5) + 1, 20); + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, strlen(key5), data5, strlen(data5) + 1, 20); const char* data6= "new Data after updated"; taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); - cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data6, strlen(data6) + 1, 20); + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, strlen(key5), data6, strlen(data6) + 1, 20); printf("%s\n", cachedObj2); taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true); const char* data7 = "add call update procedure"; - cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data7, strlen(data7) + 1, 20); + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, strlen(key5), data7, strlen(data7) + 1, 20); printf("%s\n=======================================\n\n", cachedObj2); - char* cc = (char*) taosCacheAcquireByName(tscCacheHandle, key5); + char* cc = (char*) taosCacheAcquireByKey(tscCacheHandle, key5, strlen(key5)); taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true); taosCacheRelease(tscCacheHandle, (void**) &cc, false); @@ -76,7 +76,7 @@ TEST(testCase, client_cache_test) { const char* data8 = "ttft"; const char* key6 = "key6"; - char* ft = (char*) taosCachePut(tscCacheHandle, key6, data8, strlen(data8), 20); + char* ft = (char*) taosCachePut(tscCacheHandle, key6, strlen(key6), data8, strlen(data8), 20); taosCacheRelease(tscCacheHandle, (void**) &ft, false); /** @@ -85,7 +85,7 @@ TEST(testCase, client_cache_test) { uint64_t startTime = taosGetTimestampUs(); printf("Cache Performance Test\nstart time:%" PRIu64 "\n", startTime); for(int32_t i=0; i<1000; ++i) { - char* dd = (char*) taosCacheAcquireByName(tscCacheHandle, key6); + char* dd = (char*) taosCacheAcquireByKey(tscCacheHandle, key6, strlen(key6)); if (dd != NULL) { // printf("get the data\n"); } else { @@ -105,7 +105,7 @@ TEST(testCase, client_cache_test) { TEST(testCase, cache_resize_test) { const int32_t REFRESH_TIME_IN_SEC = 2; - auto* pCache = taosCacheInit(REFRESH_TIME_IN_SEC); + auto* pCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, false, NULL); char key[256] = {0}; char data[1024] = "abcdefghijk"; @@ -116,7 +116,7 @@ TEST(testCase, cache_resize_test) { for(int32_t i = 0; i < num; ++i) { int32_t len = sprintf(key, "abc_%7d", i); - taosCachePut(pCache, key, data, len, 3600); + taosCachePut(pCache, key, strlen(key), data, len, 3600); } uint64_t endTime = taosGetTimestampUs(); @@ -125,7 +125,7 @@ TEST(testCase, cache_resize_test) { startTime = taosGetTimestampUs(); for(int32_t i = 0; i < num; ++i) { int32_t len = sprintf(key, "abc_%7d", i); - void* k = taosCacheAcquireByName(pCache, key); + void* k = taosCacheAcquireByKey(pCache, key, len); assert(k != 0); } endTime = taosGetTimestampUs(); From f90f5d5fb337adfa78067c56d9f29209e2d86ba9 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 1 Jul 2020 09:28:11 +0000 Subject: [PATCH 18/31] rand() is changed to random() --- src/rpc/src/rpcMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 3a93975022..facfb79352 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -641,7 +641,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) { pConn->pRpc = pRpc; pConn->sid = sid; - pConn->tranId = (uint16_t)(rand() & 0xFFFF); + pConn->tranId = (uint16_t)(random() & 0xFFFF); pConn->ownId = htonl(pConn->sid); pConn->linkUid = (uint32_t)((int64_t)pConn + (int64_t)getpid()); pConn->spi = pRpc->spi; From 9f4ce03e8409e3e0d6c0892eb11464d01f69703f Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 1 Jul 2020 17:41:31 +0800 Subject: [PATCH 19/31] sort KVRow by colId --- src/client/src/tscParseInsert.c | 11 ++++++++--- src/client/src/tscSQLParser.c | 12 +++++++++--- src/common/inc/tdataformat.h | 1 + src/common/src/tdataformat.c | 16 ++++++++++++++++ 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index fd2a9f5a8c..4400ca2c25 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -918,6 +918,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { char tagVal[TSDB_MAX_TAGS_LEN]; code = tsParseOneColumnData(pSchema, &sToken, tagVal, pCmd->payload, &sql, false, tinfo.precision); if (code != TSDB_CODE_SUCCESS) { + tdDestroyKVRowBuilder(&kvRowBuilder); return code; } @@ -925,10 +926,14 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - pTag->dataLen = kvRowLen(row); - memcpy(pTag->data, row, pTag->dataLen); - free(row); tdDestroyKVRowBuilder(&kvRowBuilder); + if (row == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + tdSortKVRowByColIdx(row); + pTag->dataLen = kvRowLen(row); + kvRowCpy(pTag->data, row); + free(row); index = 0; sToken = tStrGetToken(sql, &index, false, 0, NULL); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 57ae8426ff..9f557f5529 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5634,6 +5634,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { // validate the length of binary if (pList->a[i].pVar.nLen + VARSTR_HEADER_SIZE > pSchema->bytes) { + tdDestroyKVRowBuilder(&kvRowBuilder); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } @@ -5641,6 +5642,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { char tagVal[TSDB_MAX_TAGS_LEN]; ret = tVariantDump(&(pList->a[i].pVar), tagVal, pSchema->type, true); if (ret != TSDB_CODE_SUCCESS) { + tdDestroyKVRowBuilder(&kvRowBuilder); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } @@ -5648,10 +5650,14 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { } SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - pTag->dataLen = kvRowLen(row); - memcpy(pTag->data, row, pTag->dataLen); - free(row); tdDestroyKVRowBuilder(&kvRowBuilder); + if (row == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + tdSortKVRowByColIdx(row); + pTag->dataLen = kvRowLen(row); + kvRowCpy(pTag->data, row); + free(row); // table name if (tscValidateName(&pInfo->pCreateTableInfo->name) != TSDB_CODE_SUCCESS) { diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index da2f07da04..baa212d8b7 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -290,6 +290,7 @@ SKVRow tdKVRowDup(SKVRow row); int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value); int tdEncodeKVRow(void **buf, SKVRow row); void * tdDecodeKVRow(void *buf, SKVRow *row); +void tdSortKVRowByColIdx(SKVRow row); static FORCE_INLINE int comparTagId(const void *key1, const void *key2) { if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) { diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index d7b2af8706..e5cbcfd143 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -515,6 +515,22 @@ SKVRow tdKVRowDup(SKVRow row) { return trow; } +static int compareColIdx(const void* a, const void* b) { + const SColIdx* x = (const SColIdx*)a; + const SColIdx* y = (const SColIdx*)b; + if (x->colId > y->colId) { + return 1; + } + if (x->colId < y->colId) { + return -1; + } + return 0; +} + +void tdSortKVRowByColIdx(SKVRow row) { + qsort(kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), compareColIdx); +} + int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { SColIdx *pColIdx = NULL; SKVRow row = *orow; From 72028706481972a1add2405d0bb7fc99d20b0769 Mon Sep 17 00:00:00 2001 From: dengyihao Date: Wed, 1 Jul 2020 17:59:43 +0800 Subject: [PATCH 20/31] fixbug use_after_free --- src/rpc/src/rpcMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index facfb79352..429edc412e 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1530,9 +1530,9 @@ static void rpcDecRef(SRpcInfo *pRpc) { if (atomic_sub_fetch_32(&pRpc->refCount, 1) == 0) { taosHashCleanup(pRpc->hash); - taosTmrCleanUp(pRpc->tmrCtrl); taosIdPoolCleanUp(pRpc->idPool); rpcCloseConnCache(pRpc->pCache); + taosTmrCleanUp(pRpc->tmrCtrl); tfree(pRpc->connList); pthread_mutex_destroy(&pRpc->mutex); From 308c2c95fa1c18117c6fc69ff8241434d40e81eb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jul 2020 18:24:10 +0800 Subject: [PATCH 21/31] fix client crash --- src/util/src/tcache.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 91d559a92b..a1f40b6e27 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -432,7 +432,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { int16_t ref = T_REF_DEC(pNode); uDebug("%p data released, refcnt:%d", pNode, ref); - if (_remove) { + if (_remove && (!pNode->inTrashCan)) { __cache_wr_lock(pCacheObj); if (T_REF_VAL_GET(pNode) == 0) { @@ -546,7 +546,8 @@ void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem) { if (pCacheObj->freeFp) { pCacheObj->freeFp(pElem->pData->data); } - + + uError("-------------------free obj:%p", pElem->pData); free(pElem->pData); free(pElem); } From 3afe4cc45899d61dd0436116f9fdecbadc341796 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Jul 2020 18:58:22 +0800 Subject: [PATCH 22/31] fix invalid read --- src/query/src/qExecutor.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c8207cb986..f33d739ba1 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6032,12 +6032,15 @@ void qDestroyQueryInfo(qinfo_t qHandle) { qDebug("QInfo:%p dec refCount, value:%d", pQInfo, ref); if (ref == 0) { - if (pQInfo->fn != NULL) { - assert(pQInfo->param != NULL); - pQInfo->fn(pQInfo->param); - } + _qinfo_free_fn_t fn = pQInfo->fn; + void* param = pQInfo->param; doDestoryQueryInfo(pQInfo); + if (fn != NULL) { + assert(param != NULL); + fn(param); + } + } } From 1dc0f769d2b7d66bb412ce303909947e1fa17271 Mon Sep 17 00:00:00 2001 From: dengyihao Date: Wed, 1 Jul 2020 19:42:09 +0800 Subject: [PATCH 23/31] adjust code --- src/rpc/src/rpcMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 429edc412e..e4f364d3d3 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1529,10 +1529,10 @@ static void rpcAddRef(SRpcInfo *pRpc) static void rpcDecRef(SRpcInfo *pRpc) { if (atomic_sub_fetch_32(&pRpc->refCount, 1) == 0) { - taosHashCleanup(pRpc->hash); - taosIdPoolCleanUp(pRpc->idPool); rpcCloseConnCache(pRpc->pCache); + taosHashCleanup(pRpc->hash); taosTmrCleanUp(pRpc->tmrCtrl); + taosIdPoolCleanUp(pRpc->idPool); tfree(pRpc->connList); pthread_mutex_destroy(&pRpc->mutex); From 2284be9d788d4a0c790844fb4dfcac7bef42cdeb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 1 Jul 2020 23:38:36 +0800 Subject: [PATCH 24/31] change http code --- src/plugins/http/src/httpJson.c | 2 +- src/plugins/http/src/httpResp.c | 9 +-- tests/script/general/http/grafana.sim | 14 ++--- tests/script/general/http/restful.sim | 2 +- tests/script/general/http/restful_full.sim | 32 +++++------ tests/script/general/http/telegraf.sim | 66 +++++++++++----------- tests/script/sh/deploy.sh | 4 +- tests/script/unique/http/admin.sim | 20 +++---- tests/script/unique/http/opentsdb.sim | 59 ++++++++++--------- 9 files changed, 104 insertions(+), 104 deletions(-) diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index 9276637d0e..82666826bc 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -441,7 +441,7 @@ void httpJsonPairStatus(JsonBuf* buf, int code) { } else { httpJsonPair(buf, "status", 6, "error", 5); httpJsonItemToken(buf); - httpJsonPairIntVal(buf, "code", 4, code); + httpJsonPairIntVal(buf, "code", 4, code & 0XFFFF); httpJsonItemToken(buf); if (code == TSDB_CODE_MND_DB_NOT_SELECTED) { httpJsonPair(buf, "desc", 4, "failed to create database", 23); diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index de52e10f9a..f53aff7831 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -174,9 +174,9 @@ void httpSendErrorRespWithDesc(HttpContext *pContext, int errNo, char *desc) { } if (desc == NULL) { - httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo + 1000, httpMsg[errNo]); + httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo + 5000, httpMsg[errNo]); } else { - httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo + 1000, desc); + httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo + 5000, desc); } } @@ -184,7 +184,8 @@ void httpSendErrorResp(HttpContext *pContext, int errNo) { httpSendErrorRespWith void httpSendTaosdErrorResp(HttpContext *pContext, int errCode) { int httpCode = 400; - httpSendErrorRespImp(pContext, httpCode, "Bad Request", 1000, (char*)tstrerror(errCode)); + + httpSendErrorRespImp(pContext, httpCode, "Bad Request", errCode & 0XFFFF, (char*)tstrerror(errCode)); } void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg) { @@ -200,7 +201,7 @@ void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg) { } else {} } - httpSendErrorRespImp(pContext, httpCode, "Bad Request", 1000, temp); + httpSendErrorRespImp(pContext, httpCode, "Bad Request", TSDB_CODE_TSC_INVALID_SQL & 0XFFFF, temp); } void httpSendSuccResp(HttpContext *pContext, char *desc) { diff --git a/tests/script/general/http/grafana.sim b/tests/script/general/http/grafana.sim index dac7552edf..bf2a3b3b58 100644 --- a/tests/script/general/http/grafana.sim +++ b/tests/script/general/http/grafana.sim @@ -54,43 +54,43 @@ print =============== step2 - login system_content curl 127.0.0.1:6020/grafana/ print 1-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/grafana/xx print 2-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/grafana/login/xx/xx/ print 3-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"mnode invalid user"}@ then +if $system_content != @{"status":"error","code":849,"desc":"mnode invalid user"}@ then return -1 endi system_content curl 127.0.0.1:6020/grafana/root/1/123/1/1/3 print 4-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:6020/grafana/login/1/root/1/ print 5-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"mnode invalid user"}@ then +if $system_content != @{"status":"error","code":849,"desc":"mnode invalid user"}@ then return -1 endi system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:6020/grafana/root/1/login print 6-> $system_content -if $system_content != @{"status":"error","code":1010,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":5010,"desc":"invalid type of Authorization"}@ then return -1 endi system_content curl -H 'Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:6020/grafana/root/1/login print 7-> $system_content -if $system_content != @{"status":"error","code":1010,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":5010,"desc":"invalid type of Authorization"}@ then return -1 endi diff --git a/tests/script/general/http/restful.sim b/tests/script/general/http/restful.sim index 5ee8bde46f..7f39a9c748 100644 --- a/tests/script/general/http/restful.sim +++ b/tests/script/general/http/restful.sim @@ -77,7 +77,7 @@ endi system_content curl 127.0.0.1:6020/rest/login/u2/aabcd_1234 print curl 127.0.0.1:6020/rest/login/u2/abcd_1234 -----> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then +if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then return -1 endi diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index aa667daee4..60ffa4cb28 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -14,57 +14,57 @@ print =============== step1 - login system_content curl 127.0.0.1:6020/rest/ print 1-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/rest/xx print 2-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/rest/login print 3-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi #4 system_content curl 127.0.0.1:6020/rest/login/root print 4-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/rest/login/root/123 print 5-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then +if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then return -1 endi system_content curl 127.0.0.1:6020/rest/login/root/123/1/1/3 print 6-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then +if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:6020/rest/login/root/1 print 7-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then +if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then return -1 endi #8 system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:6020/rest/login/root/1 print 8-> $system_content -if $system_content != @{"status":"error","code":1010,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":5010,"desc":"invalid type of Authorization"}@ then return -1 endi system_content curl -H 'Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:6020/rest/login/root/1 print 9-> $system_content -if $system_content != @{"status":"error","code":1010,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":5010,"desc":"invalid type of Authorization"}@ then return -1 endi @@ -93,40 +93,40 @@ endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d1' 127.0.0.1:6020/rest/sql print 13-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"mnode database aleady exist"}@ then +if $system_content != @{"status":"error","code":897,"desc":"mnode database aleady exist"}@ then return -1 endi #14 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '' 127.0.0.1:6020/rest/sql print 14-> $system_content -if $system_content != @{"status":"error","code":1012,"desc":"no sql input"}@ then +if $system_content != @{"status":"error","code":5012,"desc":"no sql input"}@ then return -1 endi #system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'use d1' 127.0.0.1:6020/rest/sql #print 15-> $system_content -#if $system_content != @{"status":"error","code":1017,"desc":"no need to execute use db cmd"}@ then +#if $system_content != @{"status":"error","code":5017,"desc":"no need to execute use db cmd"}@ then #if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then # return -1 #endi #system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' use d1' 127.0.0.1:6020/rest/sql #print 16-> $system_content -#if $system_content != @{"status":"error","code":1017,"desc":"no need to execute use db cmd"}@ then +#if $system_content != @{"status":"error","code":5017,"desc":"no need to execute use db cmd"}@ then # return -1 #endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' used1' 127.0.0.1:6020/rest/sql print 17-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"invalid SQL: invalid SQL: syntax error near 'used1'"}@ then +if $system_content != @{"status":"error","code":512,"desc":"invalid SQL: invalid SQL: syntax error near 'used1'"}@ then return -1 endi #18 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' show tables;' 127.0.0.1:6020/rest/sql print 18-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"mnode db not selected"}@ then +if $system_content != @{"status":"error","code":896,"desc":"mnode db not selected"}@ then return -1 endi @@ -147,7 +147,7 @@ print =============== step3 - db system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1;' 127.0.0.1:6020/rest/sql print 21-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"mnode invalid table name"}@ then +if $system_content != @{"status":"error","code":866,"desc":"mnode invalid table name"}@ then return -1 endi diff --git a/tests/script/general/http/telegraf.sim b/tests/script/general/http/telegraf.sim index d9f3340f06..1ed73f04ad 100644 --- a/tests/script/general/http/telegraf.sim +++ b/tests/script/general/http/telegraf.sim @@ -16,224 +16,224 @@ print =============== step1 - parse system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/ print $system_content -if $system_content != @{"status":"error","code":1022,"desc":"database name can not be null"}@ then +if $system_content != @{"status":"error","code":5022,"desc":"database name can not be null"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_admin' -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/ print $system_content -if $system_content != @{"status":"error","code":1022,"desc":"database name can not be null"}@ then +if $system_content != @{"status":"error","code":5022,"desc":"database name can not be null"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_admin' -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/d123456789012345678901234567890123456 print $system_content -if $system_content != @{"status":"error","code":1023,"desc":"database name too long"}@ then +if $system_content != @{"status":"error","code":5023,"desc":"database name too long"}@ then return -1 endi system_content curl -u root:taosdata -d '[]' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1027,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":5027,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1027,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":5027,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '[{}]' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1027,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":5027,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{"metrics": []}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1025,"desc":"metrics size is 0"}@ then +if $system_content != @{"status":"error","code":5025,"desc":"metrics size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"metrics": [{}]}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1027,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":5027,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{"metrics": 12}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1025,"desc":"metrics size is 0"}@ then +if $system_content != @{"status":"error","code":5025,"desc":"metrics size is 0"}@ then return -1 endi #system_content curl -u root:taosdata -d '{"metrics": [{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}]}' 127.0.0.1:6020/telegraf/db/root/taosdata1 #print $system_content -#if $system_content != @{"status":"error","code":1026,"desc":"metrics size can not more than 50"}@ then +#if $system_content != @{"status":"error","code":5026,"desc":"metrics size can not more than 50"}@ then # return -1 #endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1027,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":5027,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":111,"tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1028,"desc":"metric name type should be string"}@ then +if $system_content != @{"status":"error","code":5028,"desc":"metric name type should be string"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1029,"desc":"metric name length is 0"}@ then +if $system_content != @{"status":"error","code":5029,"desc":"metric name length is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1030,"desc":"metric name length too long"}@ then +if $system_content != @{"status":"error","code":5030,"desc":"metric name length too long"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"}}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1031,"desc":"timestamp not find"}@ then +if $system_content != @{"status":"error","code":5031,"desc":"timestamp not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":""}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1032,"desc":"timestamp type should be integer"}@ then +if $system_content != @{"status":"error","code":5032,"desc":"timestamp type should be integer"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":-1}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1033,"desc":"timestamp value smaller than 0"}@ then +if $system_content != @{"status":"error","code":5033,"desc":"timestamp value smaller than 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1034,"desc":"tags not find"}@ then +if $system_content != @{"status":"error","code":5034,"desc":"tags not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1035,"desc":"tags size is 0"}@ then +if $system_content != @{"status":"error","code":5035,"desc":"tags size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":"","timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1035,"desc":"tags size is 0"}@ then +if $system_content != @{"status":"error","code":5035,"desc":"tags size is 0"}@ then return -1 endi #system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor","host":"windows","instance":"1","objectname":"Processor","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata #print $system_content -#if $system_content != @{"status":"error","code":1036,"desc":"tags size too long"}@ then +#if $system_content != @{"status":"error","code":5036,"desc":"tags size too long"}@ then # return -1 #endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1035,"desc":"tags size is 0"}@ then +if $system_content != @{"status":"error","code":5035,"desc":"tags size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"":"windows"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1038,"desc":"tag name is null"}@ then +if $system_content != @{"status":"error","code":5038,"desc":"tag name is null"}@ then return -1 endi #system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host111111111111222222222222222222222":""},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 #print $system_content -#if $system_content != @{"status":"error","code":1039,"desc":"tag name length too long"}@ then +#if $system_content != @{"status":"error","code":5039,"desc":"tag name length too long"}@ then # return -1 #endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":true},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1040,"desc":"tag value type should be number or string"}@ then +if $system_content != @{"status":"error","code":5040,"desc":"tag value type should be number or string"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":""},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1041,"desc":"tag value is null"}@ then +if $system_content != @{"status":"error","code":5041,"desc":"tag value is null"}@ then return -1 endi -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"1022":"111"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 +system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"5022":"111"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1042,"desc":"table is null"}@ then +if $system_content != @{"status":"error","code":5042,"desc":"table is null"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1043,"desc":"table name length too long"}@ then +if $system_content != @{"status":"error","code":5043,"desc":"table name length too long"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1045,"desc":"fields size is 0"}@ then +if $system_content != @{"status":"error","code":5045,"desc":"fields size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1048,"desc":"field name is null"}@ then +if $system_content != @{"status":"error","code":5048,"desc":"field name is null"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":"","Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1051,"desc":"field value is null"}@ then +if $system_content != @{"status":"error","code":5051,"desc":"field value is null"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":true,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1050,"desc":"field value type should be number or string"}@ then +if $system_content != @{"status":"error","code":5050,"desc":"field value type should be number or string"}@ then return -1 endi diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index ee7093a2ca..98b402f4ba 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -113,11 +113,11 @@ echo "logDir $LOG_DIR" >> $TAOS_CFG echo "mDebugFlag 135" >> $TAOS_CFG echo "sdbDebugFlag 135" >> $TAOS_CFG echo "dDebugFlag 135" >> $TAOS_CFG -echo "vDebugFlag 143" >> $TAOS_CFG +echo "vDebugFlag 135" >> $TAOS_CFG echo "cDebugFlag 135" >> $TAOS_CFG echo "jnidebugFlag 135" >> $TAOS_CFG echo "odbcdebugFlag 135" >> $TAOS_CFG -echo "httpDebugFlag 135" >> $TAOS_CFG +echo "httpDebugFlag 143" >> $TAOS_CFG echo "monitorDebugFlag 131" >> $TAOS_CFG echo "mqttDebugFlag 131" >> $TAOS_CFG echo "qdebugFlag 135" >> $TAOS_CFG diff --git a/tests/script/unique/http/admin.sim b/tests/script/unique/http/admin.sim index 3a480b6ebe..10fad091b5 100644 --- a/tests/script/unique/http/admin.sim +++ b/tests/script/unique/http/admin.sim @@ -33,49 +33,49 @@ print =============== step1 - login system_content curl 127.0.0.1:6020/admin/ print 1-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/admin/xx print 2-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/admin/login print 3-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/admin/login/root print 4-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/admin/login/root/123 print 5-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then +if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then return -1 endi system_content curl 127.0.0.1:6020/admin/login/root/123/1/1/3 print 6-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then +if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then return -1 endi system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.' -d 'show databases' 127.0.0.1:6020/admin/login/root/1 print 7-> $system_content -if $system_content != @{"status":"error","code":1010,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":5010,"desc":"invalid type of Authorization"}@ then return -1 endi system_content curl -H 'Authorization: Taosd eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' 127.0.0.1:6020/admin/login/root/1 print 8-> $system_content -if $system_content != @{"status":"error","code":1053,"desc":"parse http auth token error"}@ then +if $system_content != @{"status":"error","code":5053,"desc":"parse http auth token error"}@ then return -1 endi @@ -105,7 +105,7 @@ endi system_content curl 127.0.0.1:6020/admin/logout print 11 -----> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi @@ -168,7 +168,7 @@ print =============== step7 - use dbs system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'use d1;' 127.0.0.1:6020/admin/all print 23-> $system_content -if $system_content != @{"status":"error","code":1017,"desc":"no need to execute use db cmd"}@ then +if $system_content != @{"status":"error","code":5017,"desc":"no need to execute use db cmd"}@ then return -1 endi diff --git a/tests/script/unique/http/opentsdb.sim b/tests/script/unique/http/opentsdb.sim index 2254303e9e..7e2400bb15 100644 --- a/tests/script/unique/http/opentsdb.sim +++ b/tests/script/unique/http/opentsdb.sim @@ -3,7 +3,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 -system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 system sh/exec.sh -n dnode1 -s start sleep 3000 @@ -14,92 +13,92 @@ print ============================ dnode1 start print =============== step1 - parse system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/ print $system_content -if $system_content != @{"status":"error","code":1057,"desc":"database name can not be null"}@ then +if $system_content != @{"status":"error","code":5057,"desc":"database name can not be null"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db123456789012345678901234567890db print $system_content -if $system_content != @{"status":"error","code":1058,"desc":"database name too long"}@ then +if $system_content != @{"status":"error","code":5058,"desc":"database name too long"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/ print $system_content -if $system_content != @{"status":"error","code":1057,"desc":"database name can not be null"}@ then +if $system_content != @{"status":"error","code":5057,"desc":"database name can not be null"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put2 print $system_content -if $system_content != @{"status":"error","code":1009,"desc":"http url parse error"}@ then +if $system_content != @{"status":"error","code":5009,"desc":"http url parse error"}@ then return -1 endi system_content curl -u root:taosdata -d '[]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1060,"desc":"metrics size is 0"}@ then +if $system_content != @{"status":"error","code":5060,"desc":"metrics size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '[' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1059,"desc":"invalid opentsdb json fromat"}@ then +if $system_content != @{"status":"error","code":5059,"desc":"invalid opentsdb json fromat"}@ then return -1 endi system_content curl -u root:taosdata -d '{}' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1060,"desc":"metrics size is 0"}@ then +if $system_content != @{"status":"error","code":5060,"desc":"metrics size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '[{}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1062,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":5062,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": 1,"timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1063,"desc":"metric name type should be string"}@ then +if $system_content != @{"status":"error","code":5063,"desc":"metric name type should be string"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1064,"desc":"metric name length is 0"}@ then +if $system_content != @{"status":"error","code":5064,"desc":"metric name length is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "ab1234567890123456789012345678ab1234567890123456789012345678","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"errors":[{"datapoint":{"metric":"ab1234567890123456789012345678ab1234567890123456789012345678","stable":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb","table":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb_lga_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"status":"error","code":-2147482101,"desc":"tsdb timestamp is out of range"}}],"failed":1,"success":0,"affected_rows":0}@ then +if $system_content != @{"errors":[{"datapoint":{"metric":"ab1234567890123456789012345678ab1234567890123456789012345678","stable":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb","table":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb_lga_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"status":"error","code":1547,"desc":"tsdb timestamp is out of range"}}],"failed":1,"success":0,"affected_rows":0}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1066,"desc":"timestamp not find"}@ then +if $system_content != @{"status":"error","code":5066,"desc":"timestamp not find"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": "2","value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1067,"desc":"timestamp type should be integer"}@ then +if $system_content != @{"status":"error","code":5067,"desc":"timestamp type should be integer"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": -1,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1068,"desc":"timestamp value smaller than 0"}@ then +if $system_content != @{"status":"error","code":5068,"desc":"timestamp value smaller than 0"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1078,"desc":"value not find"}@ then +if $system_content != @{"status":"error","code":5078,"desc":"value not find"}@ then return -1 endi @@ -107,49 +106,49 @@ endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1069,"desc":"tags not find"}@ then +if $system_content != @{"status":"error","code":5069,"desc":"tags not find"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1070,"desc":"tags size is 0"}@ then +if $system_content != @{"status":"error","code":5070,"desc":"tags size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": 0}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1070,"desc":"tags size is 0"}@ then +if $system_content != @{"status":"error","code":5070,"desc":"tags size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","group1": "1","group1": "1","group1": "1","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"errors":[{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbbbbbb","table":"sys_cpu_d_bbbbbbb_lga_1_1_1_1_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","group1":"1","group1":"1","group1":"1","group1":"1","host":"web01"},"status":"error","code":-2147482782,"desc":"failed to create table"}}],"failed":1,"success":0,"affected_rows":0}@ then +if $system_content != @{"errors":[{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbbbbbb","table":"sys_cpu_d_bbbbbbb_lga_1_1_1_1_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","group1":"1","group1":"1","group1":"1","group1":"1","host":"web01"},"status":"error","code":866,"desc":"failed to create table"}}],"failed":1,"success":0,"affected_rows":0}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"": "web01"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1073,"desc":"tag name is null"}@ then +if $system_content != @{"status":"error","code":5073,"desc":"tag name is null"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host01123456789001123456789001123456789001123456789001123456789001123456789": "01"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1074,"desc":"tag name length too long"}@ then +if $system_content != @{"status":"error","code":5074,"desc":"tag name length too long"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web011234567890011234567890011234567890011234567890011234567890011234567890011234567890011234567890"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1077,"desc":"tag value can not more than 64"}@ then +if $system_content != @{"status":"error","code":5077,"desc":"tag value can not more than 64"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": ""}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1076,"desc":"tag value is null"}@ then +if $system_content != @{"status":"error","code":5076,"desc":"tag value is null"}@ then return -1 endi @@ -175,11 +174,11 @@ if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09- endi print =============== step3 - multi-query data -system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846401000,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}},{"metric": "sys_cpu","timestamp": 1346846402000,"value": 18,"tags": {"host": "web02","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put +system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846405000,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}},{"metric": "sys_cpu","timestamp": 1346846402000,"value": 18,"tags": {"host": "web02","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"errors":[{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbb","table":"sys_cpu_d_bbb_lga_1_web01","timestamp":1346846401000,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"affected_rows":1,"status":"succ"}},{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbb","table":"sys_cpu_d_bbb_lga_1_web02","timestamp":1346846402000,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web02"},"affected_rows":1,"status":"succ"}}],"failed":0,"success":2,"affected_rows":2}@ then +if $system_content != @{"errors":[{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbb","table":"sys_cpu_d_bbb_lga_1_web01","timestamp":1346846405000,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"affected_rows":1,"status":"succ"}},{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbb","table":"sys_cpu_d_bbb_lga_1_web02","timestamp":1346846402000,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web02"},"affected_rows":1,"status":"succ"}}],"failed":0,"success":2,"affected_rows":2}@ then return -1 endi @@ -187,7 +186,7 @@ system_content curl -u root:taosdata -d 'select * from db.sys_cpu_d_bbb_lga_1_w print $system_content -if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09-05 20:00:00.000",18.000000000],["2012-09-05 20:00:01.000",18.000000000]],"rows":2}@ then +if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09-05 20:00:00.000",18.000000000],["2012-09-05 20:00:05.000",18.000000000]],"rows":2}@ then return -1 endi @@ -200,7 +199,7 @@ if $system_content != @{"status":"succ","head":["count(*)"],"data":[[3]],"rows": endi print =============== step4 - summary-put data -system_content curl -u root:taosdata -d '[{"metric": "sys_mem","timestamp": 1346846400000,"value": 8,"tags": {"host": "web01","group1": "1","dc": "lga"}},{"metric": "sys_mem","timestamp": 1346846401000,"value": 9,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put?details=false +system_content curl -u root:taosdata -d '[{"metric": "sys_mem","timestamp": 1346846400000,"value": 8,"tags": {"host": "web01","group1": "1","dc": "lga"}},{"metric": "sys_mem","timestamp": 1346846405000,"value": 9,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put?details=false print $system_content @@ -212,7 +211,7 @@ system_content curl -u root:taosdata -d 'select * from db.sys_mem_d_bbb_lga_1_w print $system_content -if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09-05 20:00:00.000",8.000000000],["2012-09-05 20:00:01.000",9.000000000]],"rows":2}@ then +if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09-05 20:00:00.000",8.000000000],["2012-09-05 20:00:05.000",9.000000000]],"rows":2}@ then return -1 endi @@ -234,7 +233,7 @@ system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 134 system_content curl -u root:taosdata -d 'select count(*) from db.sys_cpu_d_bbb' 127.0.0.1:6020/rest/sql/ print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"data":[[8]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"data":[[7]],"rows":1}@ then return -1 endi From a9ef9ff97e594a451fab8bf4c38684cca75c9232 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 2 Jul 2020 01:13:21 +0800 Subject: [PATCH 25/31] [td-822] fix compiler error --- src/util/src/tcache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index a09185b8ad..d763472a12 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -255,7 +255,7 @@ SCacheObj *taosCacheInitWithCb(int32_t keyType, int64_t refreshTimeInSeconds, bo return NULL; } - pthread_attr_t thattr = {0}; + pthread_attr_t thattr = {{0}}; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); From 919adac963b068ba2b38fee6d500d18d27e6bd74 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 2 Jul 2020 10:21:11 +0800 Subject: [PATCH 26/31] add general/parser/where.sim to regression test. --- tests/script/regressionSuite.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/regressionSuite.sim b/tests/script/regressionSuite.sim index f4ffb5ea9e..ff1f9f5355 100644 --- a/tests/script/regressionSuite.sim +++ b/tests/script/regressionSuite.sim @@ -124,7 +124,7 @@ run general/parser/slimit.sim run general/parser/fill.sim run general/parser/fill_stb.sim run general/parser/interp.sim -# run general/parser/where.sim +run general/parser/where.sim run general/parser/join.sim run general/parser/join_multivnode.sim run general/parser/select_with_tags.sim From cf0c2418c5195feea5f2d1bc4cfaae33c56211d9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 2 Jul 2020 10:27:50 +0800 Subject: [PATCH 27/31] fix a typo --- src/tsdb/src/tsdbMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 38b5f252b4..4b9e977a1b 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -417,7 +417,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { if (pCfg->maxTables == -1) { pCfg->maxTables = TSDB_DEFAULT_TABLES+1; } else { - if (pCfg->maxTables - 1 < TSDB_MIN_TABLES || pCfg->maxTables - 1 > TSDB_MAX_TABLES + 1) { + if (pCfg->maxTables - 1 < TSDB_MIN_TABLES || pCfg->maxTables - 1 > TSDB_MAX_TABLES) { tsdbError("vgId:%d invalid maxTables configuration! maxTables %d TSDB_MIN_TABLES %d TSDB_MAX_TABLES %d", pCfg->tsdbId, pCfg->maxTables - 1, TSDB_MIN_TABLES, TSDB_MAX_TABLES); goto _err; From 7f29f0c6d646358a601bccabad6b867602a28013 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 2 Jul 2020 11:54:03 +0800 Subject: [PATCH 28/31] invalid cfg msg while drop vnode --- src/mnode/inc/mnodeDef.h | 3 ++- src/mnode/src/mnodeDnode.c | 1 - src/mnode/src/mnodeVgroup.c | 8 +++++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index 46d2675705..b40080759f 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -135,7 +135,8 @@ typedef struct SVgObj { char dbName[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; int8_t inUse; int8_t accessState; - int8_t reserved0[5]; + int8_t status; + int8_t reserved0[4]; SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; int8_t reserved1[7]; int8_t updateEnd[1]; diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 7e7c12cf08..c1b8256a06 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -367,7 +367,6 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { pAccess++; mnodeDecVgroupRef(pVgroup); } - } if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 3855de4101..cf1796ffb4 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -38,6 +38,11 @@ #include "mnodeVgroup.h" #include "mnodePeer.h" +typedef enum { + TAOS_VG_STATUS_READY, + TAOS_VG_STATUS_DROPPING +} EVgroupStatus; + static void *tsVgroupSdb = NULL; static int32_t tsVgUpdateSize = 0; @@ -279,7 +284,7 @@ void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t o pNextV++; } - if (i == openVnodes) { + if (i == openVnodes && pVgroup->status != TAOS_VG_STATUS_READY) { mnodeSendCreateVgroupMsg(pVgroup, NULL); } @@ -728,6 +733,7 @@ void mnodeSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) { } static void mnodeSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) { + pVgroup->status = TAOS_VG_STATUS_DROPPING; // deleting mDebug("vgId:%d, send drop all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { SRpcIpSet ipSet = mnodeGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); From 2f16c2359e0c73d223f796b44131febb9e766111 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 2 Jul 2020 13:36:49 +0800 Subject: [PATCH 29/31] fix create vnode multiple times problem --- src/mnode/src/mnodeVgroup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index cf1796ffb4..9a041aa4fd 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -284,7 +284,7 @@ void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t o pNextV++; } - if (i == openVnodes && pVgroup->status != TAOS_VG_STATUS_READY) { + if (i == openVnodes && pVgroup->status == TAOS_VG_STATUS_READY) { mnodeSendCreateVgroupMsg(pVgroup, NULL); } From d7d5e33f75853d95b5e0175ca1338abf0428e007 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 2 Jul 2020 18:06:02 +0800 Subject: [PATCH 30/31] fix compile error on MEM_CHECK=true [TD-826] --- src/util/src/tmem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/src/tmem.c b/src/util/src/tmem.c index ec5f90990b..9c512ad8dc 100644 --- a/src/util/src/tmem.c +++ b/src/util/src/tmem.c @@ -193,7 +193,7 @@ static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint3 return malloc_detect_leak(size, file, line); } - SMemBlock* blk = ((char*)ptr) - sizeof(SMemBlock); + SMemBlock* blk = (SMemBlock *)((char*)ptr) - sizeof(SMemBlock); if (blk->magic != MEMBLK_MAGIC) { if (fpAllocLog != NULL) { fprintf(fpAllocLog, "%s:%d: memory is allocated by default allocator.\n", file, line); From d5b1978e9ff4a142ffa0ade5a7b977ee85493626 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 3 Jul 2020 00:21:02 +0800 Subject: [PATCH 31/31] fix importDataLastSub.py failed with valgrind. [TD-827] --- .../pytest/import_merge/importDataLastSub.py | 2 ++ tests/pytest/test.py | 15 ++++++++++--- tests/pytest/util/dnodes.py | 22 ++++++++++++++----- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/tests/pytest/import_merge/importDataLastSub.py b/tests/pytest/import_merge/importDataLastSub.py index 5c2069c90f..bfcad2d252 100644 --- a/tests/pytest/import_merge/importDataLastSub.py +++ b/tests/pytest/import_merge/importDataLastSub.py @@ -32,6 +32,7 @@ class TDTestCase: tdDnodes.stop(1) tdDnodes.deploy(1) tdDnodes.start(1) + tdLog.sleep(5) tdSql.execute('reset query cache') tdSql.execute('drop database if exists db') @@ -61,6 +62,7 @@ class TDTestCase: tdDnodes.stop(1) tdLog.sleep(5) tdDnodes.start(1) + tdLog.sleep(5) tdLog.info("================= step5") tdLog.info("import 10 data totally repetitive") diff --git a/tests/pytest/test.py b/tests/pytest/test.py index 5b35563e1b..a9da8e5671 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -81,7 +81,7 @@ if __name__ == "__main__": else: toBeKilled = "valgrind.bin" - killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -HUP " % toBeKilled + killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -HUP > /dev/null 2>&1" % toBeKilled psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled processID = subprocess.check_output(psCmd, shell=True) @@ -91,8 +91,17 @@ if __name__ == "__main__": time.sleep(1) processID = subprocess.check_output(psCmd, shell=True) - fuserCmd = "fuser -k -n tcp 6030" - os.system(fuserCmd) + for port in range(6030, 6041): + usePortPID = "lsof -i tcp:%d | grep LISTEn | awk '{print $2}'" % port + processID = subprocess.check_output(usePortPID, shell=True) + + if processID: + killCmd = "kill -9 %s" % processID + os.system(killCmd) + fuserCmd = "fuser -k -n tcp %d" % port + os.system(fuserCmd) + if valgrind: + time.sleep(2) tdLog.info('stop All dnodes') sys.exit(0) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index e24af473f3..370af1ba13 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -251,11 +251,16 @@ class TDDnode: psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -INT %s" % processID + killCmd = "kill -INT %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( psCmd, shell=True).decode("utf-8") + for port in range(6030, 6041): + fuserCmd = "fuser -k -n tcp %d" % port + os.system(fuserCmd) + if self.valgrind: + time.sleep(2) self.running = 0 tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) @@ -272,11 +277,16 @@ class TDDnode: psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s" % processID + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( psCmd, shell=True).decode("utf-8") + for port in range(6030, 6041): + fuserCmd = "fuser -k -n tcp %d" % port + os.system(fuserCmd) + if self.valgrind: + time.sleep(2) self.running = 0 tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index)) @@ -325,7 +335,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s" % processID + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -334,7 +344,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s" % processID + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -440,7 +450,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s" % processID + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -449,7 +459,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s" % processID + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output(