From 2383e1e0c85ecc2eec560eeba5219f77e734b86b Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 5 May 2020 01:37:49 +0800 Subject: [PATCH 01/17] [td-168] fix bug in show --- src/mnode/src/mgmtUser.c | 15 ++++++++------- src/mnode/src/mgmtVgroup.c | 24 ++++++++++++++---------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 787459d667..b4dd58cb3b 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -20,6 +20,7 @@ #include "tutil.h" #include "tglobal.h" #include "tgrant.h" +#include "tdataformat.h" #include "dnode.h" #include "mgmtDef.h" #include "mgmtLog.h" @@ -256,13 +257,13 @@ static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCon int32_t cols = 0; SSchema *pSchema = pMeta->schema; - pShow->bytes[cols] = TSDB_USER_LEN; + pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 6; + pShow->bytes[cols] = 8 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "privilege"); pSchema[cols].bytes = htons(pShow->bytes[cols]); @@ -270,7 +271,7 @@ static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCon pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); + strcpy(pSchema[cols].name, "create_time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -303,16 +304,16 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void cols = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, pUser->user); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pUser->user, TSDB_USER_LEN); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; if (pUser->superAuth) { - strcpy(pWrite, "super"); + STR_WITH_SIZE_TO_VARSTR(pWrite, "super", 5); } else if (pUser->writeAuth) { - strcpy(pWrite, "write"); + STR_WITH_SIZE_TO_VARSTR(pWrite, "writable", 8); } else { - strcpy(pWrite, "read"); + STR_WITH_SIZE_TO_VARSTR(pWrite, "readable", 8); } cols++; diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index d8007d000d..2162230606 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -23,6 +23,7 @@ #include "ttime.h" #include "tbalance.h" #include "tglobal.h" +#include "tdataformat.h" #include "mgmtDef.h" #include "mgmtLog.h" #include "mgmtDb.h" @@ -374,9 +375,9 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 9; + pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "vgroup status"); + strcpy(pSchema[cols].name, "vgroup_status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -409,13 +410,13 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 40; + pShow->bytes[cols] = 40 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "end point"); + strcpy(pSchema[cols].name, "end_point"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 9; + pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "vstatus"); pSchema[cols].bytes = htons(pShow->bytes[cols]); @@ -475,7 +476,8 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, pVgroup->status ? "updating" : "ready"); + char* status = pVgroup->status? "updating" : "ready"; + STR_TO_VARSTR(pWrite, status); cols++; for (int32_t i = 0; i < maxReplica; ++i) { @@ -487,18 +489,20 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo if (pDnode != NULL) { pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strncpy(pWrite, pDnode->dnodeEp, pShow->bytes[cols]-1); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols] - VARSTR_HEADER_SIZE); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, mgmtGetMnodeRoleStr(pVgroup->vnodeGid[i].role)); + status = mgmtGetMnodeRoleStr(pVgroup->vnodeGid[i].role); + STR_TO_VARSTR(pWrite, status); cols++; } else { pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, "null"); + STR_WITH_SIZE_TO_VARSTR(pWrite, "NULL", 4); cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, "null"); + STR_WITH_SIZE_TO_VARSTR(pWrite, "NULL", 4); cols++; } } From 64a2c841bc2df9cf180428b725923eda0423d39f Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 5 May 2020 02:25:57 +0800 Subject: [PATCH 02/17] [td-168] fix bug in group by tbname --- src/client/src/tscFunctionImpl.c | 21 ++++++++++++++++++--- src/client/src/tscSQLParser.c | 2 +- src/client/src/tscUtil.c | 2 +- src/query/src/queryExecutor.c | 4 ++-- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index caaedca1f2..9995f94dbe 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2965,14 +2965,28 @@ static void tag_project_function(SQLFunctionCtx *pCtx) { assert(pCtx->inputBytes == pCtx->outputBytes); for (int32_t i = 0; i < pCtx->size; ++i) { - tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType); + char* output = pCtx->aOutputBuf; + + if (pCtx->tag.nType == TSDB_DATA_TYPE_BINARY || pCtx->tag.nType == TSDB_DATA_TYPE_NCHAR) { + *(int16_t*) output = pCtx->tag.nLen; + output += VARSTR_HEADER_SIZE; + } + + tVariantDump(&pCtx->tag, output, pCtx->outputType); pCtx->aOutputBuf += pCtx->outputBytes; } } static void tag_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { INC_INIT_VAL(pCtx, 1); - tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->tag.nType); + + char* output = pCtx->aOutputBuf; + if (pCtx->tag.nType == TSDB_DATA_TYPE_BINARY || pCtx->tag.nType == TSDB_DATA_TYPE_NCHAR) { + *(int16_t*) output = pCtx->tag.nLen; + output += VARSTR_HEADER_SIZE; + } + + tVariantDump(&pCtx->tag, output, pCtx->tag.nType); pCtx->aOutputBuf += pCtx->outputBytes; } @@ -3007,7 +3021,8 @@ static void tag_function_f(SQLFunctionCtx *pCtx, int32_t index) { *(int16_t*) output = pCtx->tag.nLen; output += VARSTR_HEADER_SIZE; } - tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->tag.nType); + + tVariantDump(&pCtx->tag, output, pCtx->tag.nType); } static void copy_function(SQLFunctionCtx *pCtx) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index cf4aa3a50d..e23086c4e7 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5132,7 +5132,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) { int16_t colIndex = pColIndex->colIndex; if (colIndex == TSDB_TBNAME_COLUMN_INDEX) { type = TSDB_DATA_TYPE_BINARY; - bytes = TSDB_TABLE_NAME_LEN; + bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; // todo extract method name = TSQL_TBNAME_L; } else { if (TSDB_COL_IS_TAG(pColIndex->flag)) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index d07185b0ba..4844ab3db8 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2134,7 +2134,7 @@ char* tscGetResultColumnChr(SSqlRes* pRes, SQueryInfo* pQueryInfo, int32_t colum if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { int32_t realLen = varDataLen(pData); if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor - *(char*) (pData + realLen + sizeof(int16_t)) = 0; + *(char*) (pData + realLen + VARSTR_HEADER_SIZE) = 0; } return pData + VARSTR_HEADER_SIZE; // head is the length of binary/nchar data diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index ed0da791dc..e8abe9d819 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -1427,7 +1427,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order int32_t index = pSqlFuncMsg->colInfo.colIndex; if (TSDB_COL_IS_TAG(pIndex->flag)) { if (pIndex->colId == TSDB_TBNAME_COLUMN_INDEX) { - pCtx->inputBytes = TSDB_TABLE_NAME_LEN; + pCtx->inputBytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pCtx->inputType = TSDB_DATA_TYPE_BINARY; } else { pCtx->inputBytes = pQuery->tagColList[index].bytes; @@ -5528,7 +5528,7 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo bytes = tDataTypeDesc[type].nSize; } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { // parse the normal column type = TSDB_DATA_TYPE_BINARY; - bytes = TSDB_TABLE_NAME_LEN; + bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; } else{ int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); assert(j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags); From e06ab2ea8348cbff2fd7425471cc5036a9774a9a Mon Sep 17 00:00:00 2001 From: hzcheng Date: Tue, 5 May 2020 09:42:01 +0800 Subject: [PATCH 03/17] fix #1794 --- src/common/src/tdataformat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 02351f7d3a..28f42f85a5 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -424,8 +424,8 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) { dataColAppendVal(target->cols + j, tdGetColDataOfRow(source->cols + j, i), target->numOfPoints, target->maxPoints); } + target->numOfPoints++; } - target->numOfPoints++; } else { pTarget = tdDupDataCols(target, true); if (pTarget == NULL) goto _err; From 76ab7c06874a6070461b5bc8aca199ebff5a39d7 Mon Sep 17 00:00:00 2001 From: jtao1735 Date: Tue, 5 May 2020 01:51:25 +0000 Subject: [PATCH 04/17] peerIp is not set in pConn --- src/rpc/src/rpcMain.c | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index a6d37d6978..d463bc3d95 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -32,8 +32,6 @@ #include "rpcCache.h" #include "rpcTcp.h" #include "rpcHead.h" -#include "shash.h" - #define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) #define rpcHeadFromCont(cont) ((SRpcHead *) (cont - sizeof(SRpcHead))) @@ -262,9 +260,7 @@ void *rpcOpen(const SRpcInit *pInit) { } if (pRpc->connType == TAOS_CONN_SERVER) { - pRpc->hash = taosInitStrHash(pRpc->sessions, sizeof(pRpc), taosHashString); - -// pRpc->hash = taosHashInit(pRpc->sessions, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true); + pRpc->hash = taosHashInit(pRpc->sessions, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true); if (pRpc->hash == NULL) { tError("%s failed to init string hash", pRpc->label); rpcClose(pRpc); @@ -298,8 +294,7 @@ void rpcClose(void *param) { (*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle); (*taosCleanUpConn[pRpc->connType])(pRpc->udphandle); -// taosHashCleanup(pRpc->hash); - taosCleanUpStrHash(pRpc->hash); + taosHashCleanup(pRpc->hash); taosTmrCleanUp(pRpc->tmrCtrl); taosIdPoolCleanUp(pRpc->idPool); rpcCloseConnCache(pRpc->pCache); @@ -548,9 +543,8 @@ static void rpcCloseConn(void *thandle) { if ( pRpc->connType == TAOS_CONN_SERVER) { char hashstr[40] = {0}; - /*size_t size = */sprintf(hashstr, "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType); -// taosHashRemove(pRpc->hash, hashstr, size); - taosDeleteStrHash(pRpc->hash, hashstr); + size_t size = sprintf(hashstr, "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType); + taosHashRemove(pRpc->hash, hashstr, size); rpcFreeMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg pConn->pRspMsg = NULL; @@ -599,12 +593,10 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { char hashstr[40] = {0}; SRpcHead *pHead = (SRpcHead *)pRecv->msg; - /*size_t size = */sprintf(hashstr, "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType); + size_t size = sprintf(hashstr, "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType); // check if it is already allocated - SRpcConn **ppConn = (SRpcConn **)(taosGetStrHashData(pRpc->hash, hashstr)); - -// SRpcConn **ppConn = (SRpcConn **)(taosHashGet(pRpc->hash, hashstr, size)); + SRpcConn **ppConn = (SRpcConn **)(taosHashGet(pRpc->hash, hashstr, size)); if (ppConn) pConn = *ppConn; if (pConn) return pConn; @@ -638,10 +630,7 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { pConn->localPort = (pRpc->localPort + pRpc->index); } - taosAddStrHash(pRpc->hash, hashstr, (char *)&pConn); - -// taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES); - + taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES); tTrace("%s %p, rpc connection is allocated, sid:%d id:%s port:%u", pRpc->label, pConn, sid, pConn->user, pConn->localPort); } @@ -803,6 +792,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) { sid = pConn->sid; pConn->chandle = pRecv->chandle; + pConn->peerIp = pRecv->ip; if (pConn->peerPort == 0) pConn->peerPort = pRecv->port; if (pHead->port) pConn->peerPort = htons(pHead->port); From f0fddd5ff8b828618a80811d689607e50b4b4576 Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 5 May 2020 11:41:22 +0800 Subject: [PATCH 05/17] support binary in show msg --- src/mnode/src/mgmtDb.c | 6 ++-- src/mnode/src/mgmtDnode.c | 16 +++++------ src/mnode/src/mgmtMnode.c | 2 +- tests/script/unique/cluster/balance1.sim | 36 +++++++----------------- 4 files changed, 22 insertions(+), 38 deletions(-) diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 9cc7e200d1..9ddf647d74 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -480,7 +480,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) } #endif - pShow->bytes[cols] = 24; + pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "keep1,keep2,keep(D)"); pSchema[cols].bytes = htons(pShow->bytes[cols]); @@ -540,13 +540,13 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) } #endif - pShow->bytes[cols] = 3; + pShow->bytes[cols] = 3 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "precision"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 10; + pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 4e84a282b3..b041d862cf 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -491,7 +491,7 @@ static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 40; + pShow->bytes[cols] = 40 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "end_point"); pSchema[cols].bytes = htons(pShow->bytes[cols]); @@ -509,7 +509,7 @@ static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 12; + pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); @@ -609,19 +609,19 @@ static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 40; + pShow->bytes[cols] = 40 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "end point"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 8; + pShow->bytes[cols] = 8 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "module"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 8; + pShow->bytes[cols] = 8 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); @@ -713,13 +713,13 @@ static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC SSchema *pSchema = pMeta->schema; - pShow->bytes[cols] = TSDB_CFG_OPTION_LEN; + pShow->bytes[cols] = TSDB_CFG_OPTION_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "config name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = TSDB_CFG_VALUE_LEN; + pShow->bytes[cols] = TSDB_CFG_VALUE_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "config value"); pSchema[cols].bytes = htons(pShow->bytes[cols]); @@ -806,7 +806,7 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 12; + pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index 93da35b939..f28e286f79 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -295,7 +295,7 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 12; + pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "role"); pSchema[cols].bytes = htons(pShow->bytes[cols]); diff --git a/tests/script/unique/cluster/balance1.sim b/tests/script/unique/cluster/balance1.sim index 5c34358e58..a7113b4535 100644 --- a/tests/script/unique/cluster/balance1.sim +++ b/tests/script/unique/cluster/balance1.sim @@ -438,22 +438,25 @@ sleep 1000 print ============================== step17 print ========= check data +sql reset query cache +sleep 1000 + sql use c_b1_d1 sql select * from c_b1_t1 -if $rows != 0 then +if $rows != 5 then return -1 endi sql use c_b1_d2 sql select * from c_b1_t2 -if $rows == 0 then +if $rows == 6 then return -1 endi sql use c_b1_d3 sql select * from c_b1_t3 order by t desc print $data01 $data11 $data21 $data31 $data41 -if $rows != 1 then +if $rows != 6 then return -1 endi if $data01 != 36 then @@ -540,30 +543,11 @@ if $data41 != 85 then return -1 endi -sql use c_b1_d9 -sql select * from c_b1_t9 order by t desc -print $data01 $data11 $data21 $data31 $data41 -if $data01 != 91 then - return -1 -endi -if $data11 != 92 then - return -1 -endi -if $data21 != 93 then - return -1 -endi -if $data31 != 94 then - return -1 -endi -if $data41 != 95 then - return -1 -endi - print ============================================ over -#system sh/exec_up.sh -n dnode2 -s stop -x SIGINT -#system sh/exec_up.sh -n dnode3 -s stop -x SIGINT -#system sh/exec_up.sh -n dnode4 -s stop -x SIGINT -#system sh/exec_up.sh -n dnode5 -s stop -x SIGINT +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT +system sh/exec_up.sh -n dnode3 -s stop -x SIGINT +system sh/exec_up.sh -n dnode4 -s stop -x SIGINT +system sh/exec_up.sh -n dnode5 -s stop -x SIGINT From 7ca6cfdc328ff6fb576f69c9c4c92b114f49055d Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 5 May 2020 12:26:49 +0800 Subject: [PATCH 06/17] a sdb confirm issue --- src/mnode/src/mgmtSdb.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index e8e805b10b..3717277992 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -458,6 +458,10 @@ static int sdbWrite(void *param, void *data, int type) { // for data from WAL or forward, version may be smaller if (pHead->version <= tsSdbObj.version) { pthread_mutex_unlock(&tsSdbObj.mutex); + if (type == TAOS_QTYPE_FWD && tsSdbObj.sync != NULL) { + sdbTrace("forward request is received, version:%" PRIu64 " confirm it", pHead->version); + syncConfirmForward(tsSdbObj.sync, pHead->version, TSDB_CODE_SUCCESS); + } return TSDB_CODE_SUCCESS; } else if (pHead->version != tsSdbObj.version + 1) { pthread_mutex_unlock(&tsSdbObj.mutex); From 01faf21760e08a2599b06558bada806c18562289 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 5 May 2020 13:00:46 +0800 Subject: [PATCH 07/17] fix grep mistake in test-all.sh for CI. --- tests/test-all.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/test-all.sh b/tests/test-all.sh index 4bffca1201..93bfa7a426 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -7,23 +7,28 @@ GREEN_DARK='\033[0;32m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' +echo "### run TSIM script ###" cd script -./test.sh -f basicSuite.sim 2>&1 | grep 'success\|failed\|fault' | tee out.txt +./test.sh -f basicSuite.sim 2>&1 | grep 'success\|failed\|fault' | grep -v 'default' | tee out.txt -totalSuccess=`grep -w 'success' out.txt | wc -l` +totalSuccess=`grep 'success' out.txt | wc -l` totalBasic=`grep success out.txt | grep Suite | wc -l` if [ "$totalSuccess" -gt "0" ]; then totalSuccess=`expr $totalSuccess - $totalBasic` - echo -e "${GREEN} ### Total $totalSuccess TSIM case(s) succeed! ### ${NC}" fi -totalFailed=`grep -w 'failed\|fault' out.txt | wc -l` +echo -e "${GREEN} ### Total $totalSuccess TSIM case(s) succeed! ### ${NC}" + +totalFailed=`grep 'failed\|fault' out.txt | wc -l` +echo -e "${RED} ### Total $totalFailed TSIM case(s) failed! ### ${NC}" + if [ "$totalFailed" -ne "0" ]; then - echo -e "${RED} ### Total $totalFailed TSIM case(s) failed! ### ${NC}" +# echo -e "${RED} ### Total $totalFailed TSIM case(s) failed! ### ${NC}" exit $totalFailed fi +echo "### run Python script ###" cd ../pytest if [ "$1" == "cron" ]; then From 65169b0a82ceada7fdb9823ef1a006ae1d4c64b7 Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 5 May 2020 13:37:39 +0800 Subject: [PATCH 08/17] scripts --- tests/script/general/user/user_create.sim | 9 ++++++--- tests/script/unique/account/user_create.sim | 8 +++++--- tests/tsim/src/simExe.c | 1 + 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/script/general/user/user_create.sim b/tests/script/general/user/user_create.sim index 8f0d32e7cc..4b81313695 100644 --- a/tests/script/general/user/user_create.sim +++ b/tests/script/general/user/user_create.sim @@ -49,19 +49,22 @@ sleep 2000 sql alter user read privilege read sql show users -if $data1_read != read then +print $data1_read +if $data1_read != readable then return -1 endi sql_error alter user read privilege super sql show users -if $data1_read != read then +print $data1_read +if $data1_read != readable then return -1 endi sql alter user read privilege write sql show users -if $data1_read != write then +print $data1_read +if $data1_read != writable then return -1 endi diff --git a/tests/script/unique/account/user_create.sim b/tests/script/unique/account/user_create.sim index b22cefcf21..bc4a8f6e8e 100644 --- a/tests/script/unique/account/user_create.sim +++ b/tests/script/unique/account/user_create.sim @@ -49,19 +49,21 @@ sleep 2000 sql alter user read privilege read sql show users -if $data1_read != read then +print $data1_read +if $data1_read != readable then return -1 endi sql_error alter user read privilege super sql show users -if $data1_read != read then +print $data1_read +if $data1_read != readable then return -1 endi sql alter user read privilege write sql show users -if $data1_read != write then +if $data1_read != writable then return -1 endi diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 26291ba992..aea295c563 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -733,6 +733,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: + memset(value, 0, MAX_QUERY_VALUE_LEN); memcpy(value, row[i], fields[i].bytes); value[fields[i].bytes] = 0; // snprintf(value, fields[i].bytes, "%s", (char *)row[i]); From 7dcbe810169f36d7bf6a93c8674af34a95c1c994 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Tue, 5 May 2020 15:38:20 +0800 Subject: [PATCH 09/17] add encode and decode functions --- src/util/inc/tcoding.h | 124 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 src/util/inc/tcoding.h diff --git a/src/util/inc/tcoding.h b/src/util/inc/tcoding.h new file mode 100644 index 0000000000..f3e7a52942 --- /dev/null +++ b/src/util/inc/tcoding.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef _TD_CODING_H_ +#define _TD_CODING_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +#include "tutil.h" + +const int TNUMBER = 1; +#define IS_LITTLE_ENDIAN() (*(char *)(&TNUMBER) != 0) + +static FORCE_INLINE void *taosEncodeFixed16(void *buf, uint16_t value) { + if (IS_LITTLE_ENDIAN()) { + memcpy(buf, &value, sizeof(value)); + } else { + ((char *)buf)[0] = value & 0xff; + ((char *)buf)[1] = (value >> 8) & 0xff; + } + + return POINTER_DRIFT(buf, sizeof(value)); +} + +static FORCE_INLINE void *taosEncodeFixed32(void *buf, uint32_t value) { + if (IS_LITTLE_ENDIAN()) { + memcpy(buf, &value, sizeof(value)); + } else { + ((char *)buf)[0] = value & 0xff; + ((char *)buf)[1] = (value >> 8) & 0xff; + ((char *)buf)[2] = (value >> 16) & 0xff; + ((char *)buf)[3] = (value >> 24) & 0xff; + } + + return POINTER_DRIFT(buf, sizeof(value)); +} + +static FORCE_INLINE void *taosEncodeFixed64(void *buf, uint64_t value) { + if (IS_LITTLE_ENDIAN()) { + memcpy(buf, &value, sizeof(value)); + } else { + ((char *)buf)[0] = value & 0xff; + ((char *)buf)[1] = (value >> 8) & 0xff; + ((char *)buf)[2] = (value >> 16) & 0xff; + ((char *)buf)[3] = (value >> 24) & 0xff; + ((char *)buf)[4] = (value >> 32) & 0xff; + ((char *)buf)[5] = (value >> 40) & 0xff; + ((char *)buf)[6] = (value >> 48) & 0xff; + ((char *)buf)[7] = (value >> 56) & 0xff; + } + + return POINTER_DRIFT(buf, sizeof(value)); +} + +static FORCE_INLINE void *taosDecodeFixed16(void *buf, uint16_t *value) { + if (IS_LITTLE_ENDIAN()) { + memcpy(value, buf, sizeof(*value)); + } else { + ((char *)value)[1] = ((char *)buf)[0]; + ((char *)value)[0] = ((char *)buf)[1]; + } + + return POINTER_DRIFT(buf, sizeof(*value)); +} + +static FORCE_INLINE void *taosDecodeFixed32(void *buf, uint32_t *value) { + if (IS_LITTLE_ENDIAN()) { + memcpy(value, buf, sizeof(*value)); + } else { + ((char *)value)[3] = ((char *)buf)[0]; + ((char *)value)[2] = ((char *)buf)[1]; + ((char *)value)[1] = ((char *)buf)[2]; + ((char *)value)[0] = ((char *)buf)[3]; + } + + return POINTER_DRIFT(buf, sizeof(*value)); +} + +static FORCE_INLINE void *taosDecodeFixed64(void *buf, uint64_t *value) { + if (IS_LITTLE_ENDIAN()) { + memcpy(value, buf, sizeof(*value)); + } else { + ((char *)value)[7] = ((char *)buf)[0]; + ((char *)value)[6] = ((char *)buf)[1]; + ((char *)value)[5] = ((char *)buf)[2]; + ((char *)value)[4] = ((char *)buf)[3]; + ((char *)value)[3] = ((char *)buf)[4]; + ((char *)value)[2] = ((char *)buf)[5]; + ((char *)value)[1] = ((char *)buf)[6]; + ((char *)value)[0] = ((char *)buf)[7]; + } + + return POINTER_DRIFT(buf, sizeof(*value)); +} + +// TODO +static FORCE_INLINE void *taosEncodeVariant16(void *buf, uint16_t value) {} +static FORCE_INLINE void *taosEncodeVariant32(void *buf, uint32_t value) {} +static FORCE_INLINE void *taosEncodeVariant64(void *buf, uint64_t value) {} +static FORCE_INLINE void *taosDecodeVariant16(void *buf, uint16_t *value) {} +static FORCE_INLINE void *taosDecodeVariant32(void *buf, uint32_t *value) {} +static FORCE_INLINE void *taosDecodeVariant64(void *buf, uint64_t *value) {} + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file From f6067e1b31ab5eebedb3b89bb339aa198365f228 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 5 May 2020 16:02:58 +0800 Subject: [PATCH 10/17] [td-168] fix bug in var string handling --- src/client/inc/tsclient.h | 3 ++- src/client/src/tscAsync.c | 2 +- src/client/src/tscSql.c | 13 ++++++++++-- src/client/src/tscSubquery.c | 3 ++- src/client/src/tscUtil.c | 35 +++++++++++++++++++++------------ src/common/inc/tdataformat.h | 2 +- src/dnode/src/dnodeWrite.c | 2 +- src/inc/taos.h | 8 +++++--- src/inc/taosdef.h | 5 ++++- src/kit/shell/src/shellEngine.c | 7 +++++-- 10 files changed, 54 insertions(+), 26 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 7cb9c513e5..203ba90c5f 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -280,6 +280,7 @@ typedef struct { SResRec * pGroupRec; char * data; void ** tsrow; + int32_t* length; // length for each field for current row char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) SColumnIndex * pColumnIndex; struct SLocalReducer *pLocalReducer; @@ -421,7 +422,7 @@ int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *s void tscQueueAsyncFreeResult(SSqlObj *pSql); int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); -char * tscGetResultColumnChr(SSqlRes *pRes, SQueryInfo *pQueryInfo, int32_t column, int16_t bytes); +void tscGetResultColumnChr(SSqlRes *pRes, SFieldInfo* pFieldInfo, int32_t column); extern void * pVnodeConn; extern void * tscCacheHandle; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index b64c1ed8c0..9f207936df 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -317,7 +317,7 @@ void tscProcessFetchRow(SSchedMsg *pMsg) { SFieldSupInfo* pSup = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); if (pSup->pSqlExpr != NULL) { - pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i, pSup->pSqlExpr->resBytes); + tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i); } else { // todo add } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index aff4ad525b..87247d874b 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -425,7 +425,7 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) { assert(0); for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i, 0); + tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i); } *rows = pRes->tsrow; @@ -725,6 +725,15 @@ char *taos_get_server_info(TAOS *taos) { return pObj->sversion; } +int* taos_fetch_lengths(TAOS_RES *res) { + SSqlObj* pSql = (SSqlObj* ) res; + if (pSql == NULL || pSql->signature != pSql) { + return NULL; + } + + return pSql->res.length; +} + char *taos_get_client_info() { return version; } void taos_stop_query(TAOS_RES *res) { @@ -796,7 +805,7 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { size_t xlen = 0; - for (xlen = 0; xlen <= fields[i].bytes; xlen++) { + for (xlen = 0; xlen < fields[i].bytes - VARSTR_HEADER_SIZE; xlen++) { char c = ((char *)row[i])[xlen]; if (c == 0) break; str[len++] = c; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 9aceecd377..c3d6c0f5fc 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1849,6 +1849,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { if (pRes->tsrow == NULL) { pRes->tsrow = calloc(numOfExprs, POINTER_BYTES); + pRes->length = calloc(numOfExprs, sizeof(int32_t)); } bool success = false; @@ -1967,7 +1968,7 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { SFieldSupInfo* pSup = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, i); if (pSup->pSqlExpr != NULL) { - pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i, pSup->pSqlExpr->resBytes); + tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i); } // primary key column cannot be null in interval query, no need to check diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 4844ab3db8..76afe3002d 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -210,7 +210,7 @@ bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableI return false; } - // order by column exists, not a non-ordered projection query + // order by columnIndex exists, not a non-ordered projection query return pQueryInfo->order.orderColId < 0; } @@ -219,7 +219,7 @@ bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableInde return false; } - // order by column exists, a non-ordered projection query + // order by columnIndex exists, a non-ordered projection query return pQueryInfo->order.orderColId >= 0; } @@ -286,13 +286,15 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutput; pRes->numOfCols = numOfOutput; - pRes->tsrow = calloc(POINTER_BYTES, numOfOutput); - pRes->buffer = calloc(POINTER_BYTES, numOfOutput); + pRes->tsrow = calloc(numOfOutput, POINTER_BYTES); + pRes->length = calloc(numOfOutput, sizeof(int32_t)); // todo refactor + pRes->buffer = calloc(numOfOutput, POINTER_BYTES); // not enough memory if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) { tfree(pRes->tsrow); tfree(pRes->buffer); + tfree(pRes->length); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; return pRes->code; @@ -312,6 +314,7 @@ void tscDestroyResPointerInfo(SSqlRes* pRes) { tfree(pRes->pRsp); tfree(pRes->tsrow); + tfree(pRes->length); tfree(pRes->pGroupRec); tfree(pRes->pColumnIndex); @@ -592,7 +595,7 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, } static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) { - // TODO: optimize this function + // TODO: optimize this function, handle the case while binary is not presented int len = 0; STableMeta* pTableMeta = pTableDataBlock->pTableMeta; @@ -924,7 +927,7 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); pExpr->functionId = functionId; - // set the correct column index + // set the correct columnIndex index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; } else { @@ -1063,7 +1066,7 @@ void tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) } SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { - // ignore the tbname column to be inserted into source list + // ignore the tbname columnIndex to be inserted into source list if (pColIndex->columnIndex < 0) { return NULL; } @@ -2124,11 +2127,13 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) { } } -char* tscGetResultColumnChr(SSqlRes* pRes, SQueryInfo* pQueryInfo, int32_t column, int16_t bytes) { - SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, column); - +void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) { + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, columnIndex); + assert(pInfo->pSqlExpr != NULL); + int32_t type = pInfo->pSqlExpr->resType; + int32_t bytes = pInfo->pSqlExpr->resBytes; + char* pData = ((char*) pRes->data) + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row; if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { @@ -2137,9 +2142,13 @@ char* tscGetResultColumnChr(SSqlRes* pRes, SQueryInfo* pQueryInfo, int32_t colum *(char*) (pData + realLen + VARSTR_HEADER_SIZE) = 0; } - return pData + VARSTR_HEADER_SIZE; // head is the length of binary/nchar data + pRes->tsrow[columnIndex] = pData + VARSTR_HEADER_SIZE; + pRes->length[columnIndex] = realLen; } else { - return pData; + assert(bytes == tDataTypeDesc[type].nSize); + + pRes->tsrow[columnIndex] = pData; + pRes->length[columnIndex] = bytes; } } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 78b6cb73b2..10ca75f785 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -32,7 +32,7 @@ extern "C" { #define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) do {\ char* _e = stpncpy((char*)(x) + VARSTR_HEADER_SIZE, (str), (_maxs));\ - *(VarDataLenT*)(x) = _e - (x);\ + *(VarDataLenT*)(x) = (_e - (x) - VARSTR_HEADER_SIZE);\ } while(0) #define STR_WITH_SIZE_TO_VARSTR(x, str, _size) do {\ diff --git a/src/dnode/src/dnodeWrite.c b/src/dnode/src/dnodeWrite.c index babbcf4ae8..39757c690f 100644 --- a/src/dnode/src/dnodeWrite.c +++ b/src/dnode/src/dnodeWrite.c @@ -228,7 +228,7 @@ static void dnodeHandleIdleWorker(SWriteWorker *pWorker) { int32_t num = taosGetQueueNumber(pWorker->qset); if (num > 0) { - usleep(30); + usleep(30000); sched_yield(); } else { taosFreeQall(pWorker->qall); diff --git a/src/inc/taos.h b/src/inc/taos.h index d4f1b8f48c..2f23b10a61 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -53,9 +53,9 @@ typedef enum { } TSDB_OPTION; typedef struct taosField { - char name[64]; - short bytes; - char type; + char name[64]; + short bytes; + uint8_t type; } TAOS_FIELD; #ifdef _TD_GO_DLL_ @@ -104,6 +104,8 @@ DLL_EXPORT void taos_stop_query(TAOS_RES *res); int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); int taos_validate_sql(TAOS *taos, const char *sql); +int* taos_fetch_lengths(TAOS_RES *res); + // TAOS_RES *taos_list_tables(TAOS *mysql, const char *wild); // TAOS_RES *taos_list_dbs(TAOS *mysql, const char *wild); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index aa4889ec9d..e60124cf0a 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -36,14 +36,17 @@ extern "C" { typedef int32_t VarDataOffsetT; typedef int16_t VarDataLenT; +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) + #define varDataLen(v) ((VarDataLenT *)(v))[0] #define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) #define varDataVal(v) ((void *)((char *)v + sizeof(VarDataLenT))) #define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v)) +#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE)) // this data type is internally used only in 'in' query to hold the values #define TSDB_DATA_TYPE_ARRAY (TSDB_DATA_TYPE_NCHAR + 1) -#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) + // Bytes for each type. extern const int32_t TYPE_BYTES[11]; diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index eeaeec83f2..d34588661c 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -350,6 +350,8 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { TAOS_FIELD *fields = taos_fetch_fields(result); row = taos_fetch_row(result); + int32_t* length = taos_fetch_lengths(result); + char t_str[TSDB_MAX_BYTES_PER_ROW] = "\0"; int l[TSDB_MAX_COLUMNS] = {0}; int maxLenColumnName = 0; @@ -457,7 +459,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: memset(t_str, 0, TSDB_MAX_BYTES_PER_ROW); - memcpy(t_str, row[i], fields[i].bytes); + memcpy(t_str, row[i], length[i]); /* printf("%-*s|",max(fields[i].bytes, strlen(fields[i].name)), * t_str); */ /* printf("%-*s|", l[i], t_str); */ @@ -532,7 +534,8 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: memset(t_str, 0, TSDB_MAX_BYTES_PER_ROW); - memcpy(t_str, row[i], fields[i].bytes); + memcpy(t_str, row[i], length[i]); + l[i] = MAX(fields[i].bytes, strlen(fields[i].name)); shellPrintNChar(t_str, l[i], printMode); break; From 6d6ee1eaddea23c288f637b2622714c0e5fc8c5c Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 5 May 2020 16:06:56 +0800 Subject: [PATCH 11/17] [td-168] fix bug in var string --- src/kit/shell/src/shellEngine.c | 2 +- tests/tsim/src/simExe.c | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index d34588661c..f9ea5f4777 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -613,7 +613,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: memset(t_str, 0, TSDB_MAX_BYTES_PER_ROW); - memcpy(t_str, row[i], fields[i].bytes); + memcpy(t_str, row[i], length[i]); fprintf(fp, "\'%s\'", t_str); break; case TSDB_DATA_TYPE_TIMESTAMP: diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 26291ba992..dc242b82f2 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -676,6 +676,8 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { while ((row = taos_fetch_row(result))) { if (numOfRows < MAX_QUERY_ROW_NUM) { TAOS_FIELD *fields = taos_fetch_fields(result); + int* length = taos_fetch_lengths(result); + for (int i = 0; i < num_fields; i++) { char *value = NULL; if (i < MAX_QUERY_COL_NUM) { @@ -733,8 +735,8 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - memcpy(value, row[i], fields[i].bytes); - value[fields[i].bytes] = 0; + memcpy(value, row[i], length[i]); + value[length[i]] = 0; // snprintf(value, fields[i].bytes, "%s", (char *)row[i]); break; case TSDB_DATA_TYPE_TIMESTAMP: From 547a7752ca918524195aaf17eb8174e068d857b5 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 5 May 2020 16:16:44 +0800 Subject: [PATCH 12/17] [td-168] fix bug in var string --- src/client/src/tscUtil.c | 2 ++ src/mnode/src/mgmtDb.c | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 76afe3002d..d49fff1583 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2138,6 +2138,8 @@ void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t column if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { int32_t realLen = varDataLen(pData); + assert(realLen <= bytes - VARSTR_HEADER_SIZE); + if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor *(char*) (pData + realLen + VARSTR_HEADER_SIZE) = 0; } diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 9cc7e200d1..ec6144d334 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -540,13 +540,13 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) } #endif - pShow->bytes[cols] = 3; + pShow->bytes[cols] = 3 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "precision"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 10; + pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); @@ -672,7 +672,7 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; char *prec = (pDb->cfg.precision == TSDB_TIME_PRECISION_MILLI) ? TSDB_TIME_PRECISION_MILLI_STR : TSDB_TIME_PRECISION_MICRO_STR; - strcpy(pWrite, prec); + STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; From 705dde6323d9d590f470dd381ac8ee263460a889 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 5 May 2020 16:34:28 +0800 Subject: [PATCH 13/17] [td-168] fix binary|nchar length validation bug. --- src/client/src/tscParseInsert.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 8f40cf6fb9..2ce3748dd1 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -310,7 +310,7 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, *payload = TSDB_DATA_BINARY_NULL; } else { // too long values will return invalid sql, not be truncated automatically - if (pToken->n > pSchema->bytes) { + if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { return tscInvalidSQLErrMsg(msg, "string data overflow", pToken->z); } @@ -328,7 +328,7 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, } else { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' int32_t resLen = -1; - if (!taosMbsToUcs4(pToken->z, pToken->n, payload + VARSTR_HEADER_SIZE, pSchema->bytes, &resLen)) { + if (!taosMbsToUcs4(pToken->z, pToken->n, payload + VARSTR_HEADER_SIZE, pSchema->bytes - VARSTR_HEADER_SIZE, &resLen)) { char buf[512] = {0}; snprintf(buf, 512, "%s", strerror(errno)); From e2a9b1c28dfaaa449078f1b9236988e43fdd559c Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 5 May 2020 18:40:13 +0800 Subject: [PATCH 14/17] fix binary.py and util/sql.py to support binary test [TD-167] --- tests/pytest/insert/binary.py | 12 ++++++------ tests/pytest/util/sql.py | 3 +++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/pytest/insert/binary.py b/tests/pytest/insert/binary.py index 9989865f96..53041b1ce7 100644 --- a/tests/pytest/insert/binary.py +++ b/tests/pytest/insert/binary.py @@ -27,8 +27,8 @@ class TDTestCase: tdSql.query('select speed from tb order by ts desc') tdLog.info('tdSql.checkRow(1)') tdSql.checkRows(1) - tdLog.info('tdSql.checkData(0, 0, 1234)') - tdSql.checkData(0, 0, 1234) + tdLog.info("tdSql.checkData(0, 0, '1234')") + tdSql.checkData(0, 0, '1234') tdLog.info('=============== step3') tdLog.info("insert into tb values (now+2a, '23456')") tdSql.execute("insert into tb values (now+2a, '23456')") @@ -37,8 +37,8 @@ class TDTestCase: tdLog.info('tdSql.checkRow(2)') tdSql.checkRows(2) tdLog.info('==> $data00') - tdLog.info('tdSql.checkData(0, 0, 23456)') - tdSql.checkData(0, 0, 23456) + tdLog.info("tdSql.checkData(0, 0, '23456')") +# tdSql.checkData(0, 0, '23456') tdLog.info('=============== step4') tdLog.info("insert into tb values (now+3a, '345678')") tdSql.error("insert into tb values (now+3a, '345678')") @@ -49,8 +49,8 @@ class TDTestCase: tdLog.info('tdSql.checkRow(3)') tdSql.checkRows(3) tdLog.info('==> $data00') - tdLog.info('tdSql.checkData(0, 0, 34567)') - tdSql.checkData(0, 0, 34567) + tdLog.info("tdSql.checkData(0, 0, '34567')") + tdSql.checkData(0, 0, '34567') tdLog.info('drop database db') tdSql.execute('drop database db') tdLog.info('show databases') diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index a1f7dd2f64..0e7e186206 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -93,6 +93,9 @@ class TDSql: if data is None: tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%s" % (self.sql, row, col, self.queryResult[row][col], data)) + elif isinstance(data, str): + tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%s" % + (self.sql, row, col, self.queryResult[row][col], data)) elif isinstance(data, datetime.date): tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%s" % (self.sql, row, col, self.queryResult[row][col], data)) From 638f7b4afbbda16ec6323f51500d7a233a3ed864 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 5 May 2020 18:45:05 +0800 Subject: [PATCH 15/17] fixup. --- tests/pytest/insert/binary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/insert/binary.py b/tests/pytest/insert/binary.py index 53041b1ce7..e254fb1438 100644 --- a/tests/pytest/insert/binary.py +++ b/tests/pytest/insert/binary.py @@ -38,7 +38,7 @@ class TDTestCase: tdSql.checkRows(2) tdLog.info('==> $data00') tdLog.info("tdSql.checkData(0, 0, '23456')") -# tdSql.checkData(0, 0, '23456') + tdSql.checkData(0, 0, '23456') tdLog.info('=============== step4') tdLog.info("insert into tb values (now+3a, '345678')") tdSql.error("insert into tb values (now+3a, '345678')") From d8f1671f3bedfbdbe7331f23f1e210fc597a8f45 Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 5 May 2020 22:20:08 +0800 Subject: [PATCH 16/17] change wal path --- src/mnode/src/mgmtSdb.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 3717277992..51f8984b48 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -143,7 +143,9 @@ static void *sdbGetTableFromId(int32_t tableId) { static int32_t sdbInitWal() { SWalCfg walCfg = {.commitLog = 2, .wals = 2, .keep = 1}; - tsSdbObj.wal = walOpen(tsMnodeDir, &walCfg); + char temp[TSDB_FILENAME_LEN]; + sprintf(temp, "%s/wal", tsMnodeDir); + tsSdbObj.wal = walOpen(temp, &walCfg); if (tsSdbObj.wal == NULL) { sdbError("failed to open sdb wal in %s", tsMnodeDir); return -1; @@ -195,10 +197,12 @@ static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, int32 return 0; } +#if 0 static int sdbGetWalInfo(void *ahandle, char *name, uint32_t *index) { strcpy(name, "wal0"); return 0; } +#endif static void sdbNotifyRole(void *ahandle, int8_t role) { sdbPrint("mnode role changed from %s to %s", mgmtGetMnodeRoleStr(tsSdbObj.role), mgmtGetMnodeRoleStr(role)); @@ -281,9 +285,9 @@ void sdbUpdateSync() { syncInfo.vgId = 1; syncInfo.version = sdbGetVersion(); syncInfo.syncCfg = syncCfg; - sprintf(syncInfo.path, "%s/", tsMnodeDir); - syncInfo.ahandle = NULL; - syncInfo.getWalInfo = sdbGetWalInfo; + sprintf(syncInfo.path, "%s", tsMnodeDir); + syncInfo.ahandle = tsSdbObj.wal; + syncInfo.getWalInfo = walGetWalFile; syncInfo.getFileInfo = sdbGetFileInfo; syncInfo.writeToCache = sdbWrite; syncInfo.confirmForward = sdbConfirmForward; From 54a704a467e8e0ef17e42b7f523c42b6503c8616 Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 5 May 2020 22:28:33 +0800 Subject: [PATCH 17/17] a small change --- src/mnode/src/mgmtSdb.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 51f8984b48..a55869222d 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -197,12 +197,9 @@ static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, int32 return 0; } -#if 0 static int sdbGetWalInfo(void *ahandle, char *name, uint32_t *index) { - strcpy(name, "wal0"); - return 0; + return walGetWalFile(tsSdbObj.wal, name, index); } -#endif static void sdbNotifyRole(void *ahandle, int8_t role) { sdbPrint("mnode role changed from %s to %s", mgmtGetMnodeRoleStr(tsSdbObj.role), mgmtGetMnodeRoleStr(role)); @@ -286,8 +283,8 @@ void sdbUpdateSync() { syncInfo.version = sdbGetVersion(); syncInfo.syncCfg = syncCfg; sprintf(syncInfo.path, "%s", tsMnodeDir); - syncInfo.ahandle = tsSdbObj.wal; - syncInfo.getWalInfo = walGetWalFile; + syncInfo.ahandle = NULL; + syncInfo.getWalInfo = sdbGetWalInfo; syncInfo.getFileInfo = sdbGetFileInfo; syncInfo.writeToCache = sdbWrite; syncInfo.confirmForward = sdbConfirmForward;