From 44620e056c19e62e6f5b50e72a43426d595d0b88 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 7 Apr 2020 15:57:48 +0800 Subject: [PATCH 01/78] [td-98] support iplist --- src/client/inc/tsclient.h | 2 +- src/client/src/tscSQLParser.c | 1 - src/client/src/tscServer.c | 40 +++++++++++++++++++++++------------ src/client/src/tscUtil.c | 4 +--- src/mnode/src/mgmtTable.c | 4 ++-- src/query/src/queryExecutor.c | 18 +++------------- src/vnode/tsdb/src/tsdbRead.c | 11 ++-------- 7 files changed, 36 insertions(+), 44 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index dad32e58da..80c3e77eff 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -351,7 +351,7 @@ typedef struct SSqlObj { char * sqlstr; char retry; char maxRetry; - SRpcIpSet *ipList; + SRpcIpSet ipList; char freed : 4; char listed : 4; tsem_t rspSem; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 40e70c247e..4462af7f95 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -209,7 +209,6 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } int32_t code = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); -// assert(pQueryInfo->numOfTables == 0); STableMetaInfo* pTableMetaInfo = NULL; if (pQueryInfo->numOfTables == 0) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 1e7ad937ac..102d82bcb1 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -169,17 +169,27 @@ void tscProcessActivityTimer(void *handle, void *tmrId) { } int tscSendMsgToServer(SSqlObj *pSql) { - char *pMsg = rpcMallocCont(pSql->cmd.payloadLen); + SSqlCmd* pCmd = &pSql->cmd; + + char *pMsg = rpcMallocCont(pCmd->payloadLen); if (NULL == pMsg) { tscError("%p msg:%s malloc fail", pSql, taosMsg[pSql->cmd.msgType]); return TSDB_CODE_CLI_OUT_OF_MEMORY; } - pSql->ipList->ip[0] = inet_addr(tsPrivateIp); - if (pSql->cmd.command < TSDB_SQL_MGMT) { - pSql->ipList->port = tsDnodeShellPort; - tscPrint("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList->port); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + pSql->ipList.numOfIps = pTableMeta->numOfVpeers; + pSql->ipList.port = tsDnodeShellPort; + pSql->ipList.inUse = 0; + + for(int32_t i = 0; i < pTableMeta->numOfVpeers; ++i) { + pSql->ipList.ip[i] = pTableMeta->vpeerDesc[i].ip; + } + + tscPrint("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port); memcpy(pMsg, pSql->cmd.payload + tsRpcHeadSize, pSql->cmd.payloadLen); SRpcMsg rpcMsg = { @@ -189,10 +199,12 @@ int tscSendMsgToServer(SSqlObj *pSql) { .handle = pSql, .code = 0 }; - rpcSendRequest(pVnodeConn, pSql->ipList, &rpcMsg); + rpcSendRequest(pVnodeConn, &pSql->ipList, &rpcMsg); } else { - pSql->ipList->port = tsMnodeShellPort; - tscTrace("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList->port); + pSql->ipList = tscMgmtIpList; + pSql->ipList.port = tsMnodeShellPort; + + tscTrace("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port); memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); SRpcMsg rpcMsg = { .msgType = pSql->cmd.msgType, @@ -201,7 +213,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { .handle = pSql, .code = 0 }; - rpcSendRequest(pTscMgmtConn, pSql->ipList, &rpcMsg); + rpcSendRequest(pTscMgmtConn, &pSql->ipList, &rpcMsg); } return TSDB_CODE_SUCCESS; @@ -405,7 +417,7 @@ int tscProcessSql(SSqlObj *pSql) { } // temp - pSql->ipList = &tscMgmtIpList; +// pSql->ipList = tscMgmtIpList; // if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { // pSql->index = pTableMetaInfo->pTableMeta->index; // } else { // it must be the parent SSqlObj for super table query @@ -417,7 +429,7 @@ int tscProcessSql(SSqlObj *pSql) { // } // } } else if (pSql->cmd.command < TSDB_SQL_LOCAL) { - pSql->ipList = &tscMgmtIpList; + pSql->ipList = tscMgmtIpList; } else { // local handler return (*tscProcessMsgRsp[pCmd->command])(pSql); } @@ -532,9 +544,9 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pShellMsg->numOfBlocks = htonl(pSql->cmd.numOfTablesInSubmit); // number of meters to be inserted - // pSql->cmd.payloadLen is set during copying data into paylaod + // pSql->cmd.payloadLen is set during copying data into payload pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - tscTrace("%p build submit msg, vgId:%d numOfVnodes:%d", pSql, pTableMeta->vgId, htons(pMsgDesc->numOfVnodes)); + tscTrace("%p build submit msg, vgId:%d numOfVnodes:%d", pSql, pTableMeta->vgId, htonl(pMsgDesc->numOfVnodes)); return TSDB_CODE_SUCCESS; } @@ -1836,6 +1848,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { for (int i = 0; i < TSDB_VNODES_SUPPORT; ++i) { pMetaMsg->vpeerDesc[i].vgId = htonl(pMetaMsg->vpeerDesc[i].vgId); + pMetaMsg->vpeerDesc[i].ip = htonl(pMetaMsg->vpeerDesc[i].ip); + pMetaMsg->vpeerDesc[i].dnodeId = htonl(pMetaMsg->vpeerDesc[i].dnodeId); } SSchema* pSchema = pMetaMsg->schema; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 61b7ab4ec5..dd0945ab04 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -860,12 +860,10 @@ int tscAllocPayload(SSqlCmd* pCmd, int size) { pCmd->allocSize = size; } - memset(pCmd->payload, 0, pCmd->payloadLen); + memset(pCmd->payload, 0, pCmd->allocSize); } - //memset(pCmd->payload, 0, pCmd->allocSize); assert(pCmd->allocSize >= size); - return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index c4996fc4bb..56ea02a312 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -1508,9 +1508,9 @@ static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) { for (int32_t i = 0; i < TSDB_VNODES_SUPPORT; ++i) { if (usePublicIp) { - pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].publicIp; + pMeta->vpeerDesc[i].ip = htonl(pVgroup->vnodeGid[i].publicIp); } else { - pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].privateIp; + pMeta->vpeerDesc[i].ip = htonl(pVgroup->vnodeGid[i].privateIp); } pMeta->vpeerDesc[i].vgId = htonl(pVgroup->vgId); pMeta->vpeerDesc[i].dnodeId = htonl(pVgroup->vnodeGid[i].dnodeId); diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index edb0e0aaed..dfa5dae73a 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -2480,12 +2480,11 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl } if (r == BLK_DATA_NO_NEEDED) { - // qTrace("QInfo:%p vid:%d sid:%d id:%s, slot:%d, data block ignored, brange:%" PRId64 "-%" PRId64 ", - // rows:%d", GET_QINFO_ADDR(pQuery), pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->slot, - // pBlock->keyFirst, pBlock->keyLast, pBlock->numOfPoints); + qTrace("QInfo:%p slot:%d, data block ignored, brange:%" PRId64 "-%" PRId64 ", rows:%d", + GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->size); } else if (r == BLK_DATA_FILEDS_NEEDED) { if (tsdbRetrieveDataBlockStatisInfo(pRuntimeEnv->pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { -// return DISK_DATA_LOAD_FAILED; + // return DISK_DATA_LOAD_FAILED; } if (*pStatis == NULL) { @@ -5908,14 +5907,6 @@ static void freeQInfo(SQInfo *pQInfo) { tfree(pQuery->sdata[col]); } - // for (int col = 0; col < pQuery->numOfCols; ++col) { - // vnodeFreeColumnInfo(&pQuery->colList[col].data); - // } - // - // if (pQuery->colList[0].colIdx != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - // tfree(pQuery->tsData); - // } - sem_destroy(&(pQInfo->dataReady)); teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); @@ -6125,9 +6116,6 @@ void qTableQuery(SQInfo *pQInfo) { dTrace("QInfo:%p query task is launched", pQInfo); -// sem_post(&pQInfo->dataReady); -// pQInfo->runtimeEnv.pQuery->status = QUERY_OVER; - int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); if (numOfTables == 1) { singleTableQueryImpl(pQInfo); diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 49cbd10042..a44174ae66 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -37,10 +37,6 @@ typedef struct SField { // todo need the definition } SField; -typedef struct SHeaderFileInfo { - int32_t fileId; -} SHeaderFileInfo; - typedef struct SQueryFilePos { int32_t fid; int32_t slot; @@ -380,11 +376,12 @@ static bool loadQualifiedDataFromFileBlock(STsdbQueryHandle *pQueryHandle) { SArray *sa = getDefaultLoadColumns(pQueryHandle, true); if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { - // query ended in current block if (pQueryHandle->window.ekey < pBlock->keyLast) { doLoadDataFromFileBlock(pQueryHandle); filterDataInDataBlock(pQueryHandle, pCheckInfo->pDataCols, sa); + } else { // the whole block is loaded in to buffer + pQueryHandle->realNumOfRows = pBlock->numOfPoints; } } else {// todo desc query if (pQueryHandle->window.ekey > pBlock->keyFirst) { @@ -932,10 +929,6 @@ SArray *tsdbRetrieveDataBlock(tsdb_query_handle_t *pQueryHandle, SArray *pIdList STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; if (pHandle->cur.fid < 0) { - - - - return pHandle->pColumns; } else { STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); From 54f28dc5f31f2cc22dd9897ec293fb166afc1c8e Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 7 Apr 2020 16:30:14 +0800 Subject: [PATCH 02/78] [td-98] add retry counting --- src/client/src/tscServer.c | 10 +++++++--- src/rpc/src/rpcMain.c | 4 ++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 102d82bcb1..a6c26d4b8f 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -266,11 +266,15 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { rpcFreeCont(rpcMsg->pCont); return; } else { - tscTrace("%p it shall renew meter meta, code:%d", pSql, tstrerror(rpcMsg->code)); + tscTrace("%p it shall renew table meta, code:%d", pSql, tstrerror(rpcMsg->code)); pSql->maxRetry = TSDB_VNODES_SUPPORT * 2; pSql->res.code = rpcMsg->code; // keep the previous error code - + if (++pSql->retry > pSql->maxRetry) { + tscError("%p max retry %d reached, ", pSql, pSql->retry); + return; + } + rpcMsg->code = tscRenewMeterMeta(pSql, pTableMetaInfo->name); if (pTableMetaInfo->pTableMeta) { @@ -2449,7 +2453,7 @@ static void tscWaitingForCreateTable(SSqlCmd *pCmd) { int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) { int code = 0; - // handle metric meta renew process + // handle table meta renew process SSqlCmd *pCmd = &pSql->cmd; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 6f86c2dd7c..7e6f024422 100755 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1096,6 +1096,10 @@ static void rpcProcessConnError(void *param, void *id) { SRpcInfo *pRpc = pContext->pRpc; SRpcMsg rpcMsg; + if (pRpc == NULL) { + return; + } + tTrace("%s connection error happens", pRpc->label); if ( pContext->numOfTry >= pContext->ipSet.numOfIps ) { From cfd3eb2f8a7a7a2fa9a7fc70af97d6623792a74b Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 8 Apr 2020 09:54:52 +0800 Subject: [PATCH 03/78] [TD-114] --- src/inc/mnode.h | 5 +++-- src/mnode/inc/mgmtDnode.h | 11 +++++++++-- src/mnode/inc/mgmtVgroup.h | 6 ++++-- src/mnode/src/mgmtDnode.c | 25 ++++++++++++++----------- src/mnode/src/mgmtVgroup.c | 18 +++++++++++++----- src/util/inc/tstatus.h | 28 +++------------------------- src/util/src/tstatus.c | 8 -------- 7 files changed, 46 insertions(+), 55 deletions(-) diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 3f66c46d7b..651ad0036e 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -147,9 +147,10 @@ typedef struct _vg_obj { int64_t createdTime; SVnodeGid vnodeGid[TSDB_VNODES_SUPPORT]; int32_t numOfVnodes; - int32_t lbIp; + int32_t lbDnodeId; int32_t lbTime; - int8_t lbStatus; + int8_t status; + int8_t reserved[14]; int8_t updateEnd[1]; int32_t refCount; diff --git a/src/mnode/inc/mgmtDnode.h b/src/mnode/inc/mgmtDnode.h index f262cd3c5e..9c9ddea346 100644 --- a/src/mnode/inc/mgmtDnode.h +++ b/src/mnode/inc/mgmtDnode.h @@ -21,12 +21,19 @@ extern "C" { #endif #include "mnode.h" +enum _TSDB_DN_STATUS { + TSDB_DN_STATUS_OFFLINE, + TSDB_DN_STATUS_DROPING, + TSDB_DN_STATUS_BALANCING, + TSDB_DN_STATUS_READY +}; + int32_t mgmtInitDnodes(); void mgmtCleanUpDnodes(); int32_t mgmtGetDnodesNum(); void * mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode); -void mgmtIncDnodeRef(SDnodeObj *pDnode); -void mgmtDecDnodeRef(SDnodeObj *pDnode); +void mgmtReleaseDnode(SDnodeObj *pDnode); +char* mgmtGetDnodeStatusStr(int32_t dnodeStatus); SDnodeObj* mgmtGetDnode(int32_t dnodeId); SDnodeObj* mgmtGetDnodeByIp(uint32_t ip); diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h index d0b1e0de97..b4d50a8318 100644 --- a/src/mnode/inc/mgmtVgroup.h +++ b/src/mnode/inc/mgmtVgroup.h @@ -27,10 +27,12 @@ extern "C" { int32_t mgmtInitVgroups(); void mgmtCleanUpVgroups(); SVgObj *mgmtGetVgroup(int32_t vgId); -void mgmtIncVgroupRef(SVgObj *pVgroup); -void mgmtDecVgroupRef(SVgObj *pVgroup); +void mgmtReleaseVgroup(SVgObj *pVgroup); void mgmtDropAllVgroups(SDbObj *pDropDb); +void * mgmtGetNextVgroup(void *pNode, SVgObj **pVgroup); +void mgmtUpdateVgroup(SVgObj *pVgroup); + void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb); void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle); void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle); diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index ada0bce2e9..7019d8e20a 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -42,8 +42,7 @@ extern int32_t clusterInit(); extern void clusterCleanUp(); extern int32_t clusterGetDnodesNum(); extern void * clusterGetNextDnode(void *pNode, SDnodeObj **pDnode); -extern void clusterIncDnodeRef(SDnodeObj *pDnode); -extern void clusterDecDnodeRef(SDnodeObj *pDnode); +extern void clusterReleaseDnode(SDnodeObj *pDnode); extern SDnodeObj* clusterGetDnode(int32_t dnodeId); extern SDnodeObj* clusterGetDnodeByIp(uint32_t ip); #ifndef _CLUSTER @@ -120,15 +119,9 @@ int32_t mgmtGetDnodesNum() { #endif } -void mgmtIncDnodeRef(SDnodeObj *pDnode) { +void mgmtReleaseDnode(SDnodeObj *pDnode) { #ifdef _CLUSTER - return clusterIncDnodeRef(pDnode); -#endif -} - -void mgmtDecDnodeRef(SDnodeObj *pDnode) { -#ifdef _CLUSTER - return clusterDecDnodeRef(pDnode); + return clusterReleaseDnode(pDnode); #endif } @@ -724,4 +717,14 @@ static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, voi pShow->numOfReads += numOfRows; return numOfRows; -} \ No newline at end of file +} + +char* mgmtGetDnodeStatusStr(int32_t dnodeStatus) { + switch (dnodeStatus) { + case TSDB_DN_STATUS_OFFLINE: return "offline"; + case TSDB_DN_STATUS_DROPING: return "dropping"; + case TSDB_DN_STATUS_BALANCING: return "balancing"; + case TSDB_DN_STATUS_READY: return "ready"; + default: return "undefined"; + } +} diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index bbf960203a..c16cae775c 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -190,11 +190,7 @@ int32_t mgmtInitVgroups() { return 0; } -void mgmtIncVgroupRef(SVgObj *pVgroup) { - return sdbIncRef(tsVgroupSdb, pVgroup); -} - -void mgmtDecVgroupRef(SVgObj *pVgroup) { +void mgmtReleaseVgroup(SVgObj *pVgroup) { return sdbDecRef(tsVgroupSdb, pVgroup); } @@ -202,6 +198,18 @@ SVgObj *mgmtGetVgroup(int32_t vgId) { return (SVgObj *)sdbGetRow(tsVgroupSdb, &vgId); } +void mgmtUpdateVgroup(SVgObj *pVgroup) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup, + .rowSize = tsVgUpdateSize + }; + + sdbUpdateRow(&oper); + mgmtSendCreateVgroupMsg(pVgroup, NULL); +} + SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb) { return pDb->pHead; } diff --git a/src/util/inc/tstatus.h b/src/util/inc/tstatus.h index 223ed1d52d..4fa392ca35 100644 --- a/src/util/inc/tstatus.h +++ b/src/util/inc/tstatus.h @@ -24,15 +24,6 @@ extern "C" { #include #include "taoserror.h" -enum _TSDB_VG_STATUS { - TSDB_VG_STATUS_READY = TSDB_CODE_SUCCESS, - TSDB_VG_STATUS_IN_PROGRESS = 1, //TSDB_CODE_ACTION_IN_PROGRESS, - TSDB_VG_STATUS_NO_DISK_PERMISSIONS = 73,//TSDB_CODE_NO_DISK_PERMISSIONS, - TSDB_VG_STATUS_SERVER_NO_PACE = 110, //TSDB_CODE_SERV_NO_DISKSPACE, - TSDB_VG_STATUS_SERV_OUT_OF_MEMORY = 69, //TSDB_CODE_SERV_OUT_OF_MEMORY, - TSDB_VG_STATUS_INIT_FAILED = 74, //TSDB_CODE_VG_INIT_FAILED, - TSDB_VG_STATUS_FULL = 48, //TSDB_CODE_NO_ENOUGH_DNODES, -}; enum _TSDB_DB_STATUS { TSDB_DB_STATUS_READY, @@ -75,21 +66,9 @@ enum _TSDB_MN_ROLE { TSDB_MN_ROLE_MASTER }; -enum _TSDB_DN_STATUS { - TSDB_DN_STATUS_OFFLINE, - TSDB_DN_STATUS_READY -}; - -enum _TSDB_DN_LB_STATUS { - TSDB_DN_LB_STATUS_BALANCED, - TSDB_DN_LB_STATUS_BALANCING, - TSDB_DN_LB_STATUS_OFFLINE_REMOVING, - TSDB_DN_LB_STATE_SHELL_REMOVING -}; - -enum _TSDB_VG_LB_STATUS { - TSDB_VG_LB_STATUS_READY, - TSDB_VG_LB_STATUS_UPDATE +enum _TSDB_VG_STATUS { + TSDB_VG_STATUS_READY, + TSDB_VG_STATUS_UPDATE }; enum _TSDB_VN_STREAM_STATUS { @@ -111,7 +90,6 @@ char* taosGetDbStatusStr(int32_t dbStatus); char* taosGetVnodeStatusStr(int32_t vnodeStatus); char* taosGetVnodeSyncStatusStr(int32_t vnodeSyncStatus); char* taosGetVnodeDropStatusStr(int32_t dropping); -char* taosGetDnodeStatusStr(int32_t dnodeStatus); char* taosGetDnodeLbStatusStr(int32_t dnodeBalanceStatus); char* taosGetVgroupLbStatusStr(int32_t vglbStatus); char* taosGetVnodeStreamStatusStr(int32_t vnodeStreamStatus); diff --git a/src/util/src/tstatus.c b/src/util/src/tstatus.c index e8534de30b..cfcbea0ca1 100644 --- a/src/util/src/tstatus.c +++ b/src/util/src/tstatus.c @@ -69,14 +69,6 @@ char* taosGetVnodeDropStatusStr(int32_t dropping) { } } -char* taosGetDnodeStatusStr(int32_t dnodeStatus) { - switch (dnodeStatus) { - case TSDB_DN_STATUS_OFFLINE: return "offline"; - case TSDB_DN_STATUS_READY: return "ready"; - default: return "undefined"; - } -} - char* taosGetDnodeLbStatusStr(int32_t dnodeBalanceStatus) { switch (dnodeBalanceStatus) { case TSDB_DN_LB_STATUS_BALANCED: return "balanced"; From 4f6cb328b2274ebf5efe8f10a3ef7daaf495603a Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 8 Apr 2020 16:59:28 +0800 Subject: [PATCH 04/78] change the API to fit the changes in syncCfg --- src/inc/tsync.h | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 39c116c9cb..555e0503da 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -56,16 +56,13 @@ typedef struct { } SNodesRole; typedef struct { - char label[20]; // for debug purpose - char path[128]; // path to the file - int8_t replica; // number of replications, >=1 - int8_t quorum; // number of confirms required, >=1 int32_t vgId; // vgroup ID - void *ahandle; // handle provided by APP uint64_t version; // initial version - uint32_t arbitratorIp; - SNodeInfo nodeInfo[TAOS_SYNC_MAX_REPLICA]; + SSyncCfg syncCfg; // configuration from mgmt + char path[128]; // path to the file + void *ahandle; // handle provided by APP + // if name is null, get the file from index or after, used by master // if name is provided, get the named file at the specified index, used by unsynced node // it returns the file magic number and size, if file not there, magic shall be 0. @@ -76,7 +73,7 @@ typedef struct { int (*getWalInfo)(char *name, int *index); // when a forward pkt is received, call this to handle data - int (*writeToCache)(void *ahandle, SWalHead *, int type); + int (*writeToCache)(void *ahandle, void *pHead, int type); // when forward is confirmed by peer, master call this API to notify app void (*confirmForward)(void *ahandle, void *mhandle, int32_t code); @@ -89,8 +86,8 @@ typedef void* tsync_h; tsync_h syncStart(SSyncInfo *); void syncStop(tsync_h shandle); -int syncReconfig(tsync_h shandle, SSyncInfo *); -int syncForwardToPeer(tsync_h shandle, SWalHead *pHead, void *mhandle); +int syncReconfig(tsync_h shandle, SSyncCfg *); +int syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle); void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code); void syncRecover(tsync_h shandle); // recover from other nodes: int syncGetNodesRole(tsync_h shandle, SNodesRole *); From 45b90bf90994ebaeb10be8c413378903a205a910 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 8 Apr 2020 17:04:16 +0800 Subject: [PATCH 05/78] add testcase execution in CI [TD-96] --- .travis.yml | 24 +++++++++++++++++++++--- tests/script/basicSuite.sim | 4 ++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 74ed80da3f..5cff3bc72b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,12 +41,29 @@ addons: branch_pattern: coverity_scan before_script: - - mkdir build - - cd build + - mkdir debug + - cd debug script: - cmake .. - - cmake --build . + - cmake --build . || exit $? + - |- + case $TRAVIS_OS_NAME in + linux) + cd ../tests/script + sudo ./test.sh 2>&1 | tee out.txt + cat out.txt + grep success out.txt + total_success=`grep success out.txt | wc -l` + echo "Total $total_success success" + grep failed out.txt + total_failed=`grep failed out.txt | wc -l` + echo "Total $total_failed failed" + if [ "$total_failed" -ne "0" ]; then + exit $total_failed + fi + ;; + esac # # Build Matrix @@ -58,6 +75,7 @@ matrix: packages: - build-essential - cmake + - net-tools # - os: osx # addons: diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index c95695fe00..d5892d8682 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -1,8 +1,8 @@ ################################# -#run general/table/basic1.sim -run general/table/basic2.sim +run general/table/basic1.sim +#run general/table/basic2.sim #run general/table/basic3.sim ################################## From 8568f4c4610df30b8cc68e04ebc98ba61a71fbf4 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 8 Apr 2020 20:02:29 +0800 Subject: [PATCH 06/78] [TD-82] refactor balance module --- src/dnode/src/dnodeMain.c | 2 +- src/dnode/src/dnodeMgmt.c | 1 - src/inc/mnode.h | 20 +- src/{mnode/inc/mgmtAcct.h => inc/taccount.h} | 29 +-- src/{mnode/inc/mgmtBalance.h => inc/tadmin.h} | 18 +- src/inc/taosmsg.h | 2 +- src/{mnode/inc/mgmtDnode.h => inc/tbalance.h} | 31 ++- src/inc/tcluster.h | 54 +++++ src/{mnode/inc/mgmtGrant.h => inc/tgrant.h} | 4 +- src/inc/vnode.h | 10 +- src/mnode/inc/mgmtDb.h | 7 +- src/mnode/inc/mgmtMnode.h | 13 ++ src/mnode/inc/mgmtUser.h | 3 +- src/mnode/inc/mgmtVgroup.h | 5 + src/mnode/src/mgmtAcct.c | 18 +- src/mnode/src/mgmtBalance.c | 59 ++--- src/mnode/src/mgmtDClient.c | 7 +- src/mnode/src/mgmtDServer.c | 5 +- src/mnode/src/mgmtDb.c | 39 ++-- src/mnode/src/mgmtDnode.c | 210 +++++++----------- src/mnode/src/mgmtGrant.c | 2 +- src/mnode/src/mgmtMain.c | 16 +- src/mnode/src/mgmtMnode.c | 22 +- src/mnode/src/mgmtProfile.c | 12 +- src/mnode/src/mgmtShell.c | 43 +++- src/mnode/src/mgmtTable.c | 58 +++-- src/mnode/src/mgmtUser.c | 28 +-- src/mnode/src/mgmtVgroup.c | 48 ++-- src/query/src/queryExecutor.c | 1 - src/util/inc/tstatus.h | 105 --------- src/util/src/tstatus.c | 149 ------------- src/vnode/main/inc/vnodeInt.h | 10 +- src/vnode/main/src/vnodeMain.c | 13 +- src/vnode/main/src/vnodeRead.c | 2 +- src/vnode/main/src/vnodeWrite.c | 9 +- 35 files changed, 423 insertions(+), 632 deletions(-) rename src/{mnode/inc/mgmtAcct.h => inc/taccount.h} (56%) rename src/{mnode/inc/mgmtBalance.h => inc/tadmin.h} (72%) rename src/{mnode/inc/mgmtDnode.h => inc/tbalance.h} (57%) create mode 100644 src/inc/tcluster.h rename src/{mnode/inc/mgmtGrant.h => inc/tgrant.h} (95%) delete mode 100644 src/util/inc/tstatus.h delete mode 100644 src/util/src/tstatus.c diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index e2de9bf586..d95a344058 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -28,7 +28,7 @@ #include "dnodeRead.h" #include "dnodeShell.h" #include "dnodeWrite.h" -#include "mgmtGrant.h" +#include "tgrant.h" static int32_t dnodeInitSystem(); static int32_t dnodeInitStorage(); diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 5f1e7a7a94..fa3ad946e3 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -20,7 +20,6 @@ #include "taosmsg.h" #include "tlog.h" #include "trpc.h" -#include "tstatus.h" #include "tsdb.h" #include "ttime.h" #include "ttimer.h" diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 70c7f8003d..903b172068 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -21,7 +21,6 @@ extern "C" { #endif #include "os.h" - #include "taosdef.h" #include "taosmsg.h" #include "taoserror.h" @@ -40,7 +39,8 @@ extern "C" { struct _vg_obj; struct _db_obj; -struct _acctObj; +struct _acct_obj; +struct _user_obj; typedef struct { int32_t mnodeId; @@ -65,7 +65,7 @@ typedef struct { void *pSync; } SMnodeObj; -typedef struct { +typedef struct _dnode_obj { int32_t dnodeId; uint32_t privateIp; uint32_t publicIp; @@ -79,16 +79,15 @@ typedef struct { uint16_t slot; uint16_t numOfCores; // from dnode status msg int8_t alternativeRole; // from dnode status msg, 0-any, 1-mgmt, 2-dnode - int8_t lbStatus; // set in balance function - float lbScore; // calc in balance function + int8_t status; // set in balance function int32_t customScore; // config by user char dnodeName[TSDB_DNODE_NAME_LEN + 1]; int8_t reserved[15]; int8_t updateEnd[1]; int32_t refCount; SVnodeLoad vload[TSDB_MAX_VNODES]; - int32_t status; uint32_t lastReboot; // time stamp for last reboot + float score; // calc in balance function float diskAvailable; // from dnode status msg int16_t diskAvgUsage; // calc from sys.disk int16_t cpuAvgUsage; // calc from sys.cpu @@ -150,7 +149,6 @@ typedef struct _vg_obj { int32_t lbDnodeId; int32_t lbTime; int8_t status; - int8_t reserved[14]; int8_t updateEnd[1]; int32_t refCount; @@ -163,7 +161,7 @@ typedef struct _vg_obj { typedef struct _db_obj { char name[TSDB_DB_NAME_LEN + 1]; - int8_t dirty; + int8_t status; int64_t createdTime; SDbCfg cfg; int8_t reserved[15]; @@ -174,7 +172,7 @@ typedef struct _db_obj { int32_t numOfSuperTables; SVgObj *pHead; SVgObj *pTail; - struct _acctObj *pAcct; + struct _acct_obj *pAcct; } SDbObj; typedef struct _user_obj { @@ -187,7 +185,7 @@ typedef struct _user_obj { int8_t reserved[13]; int8_t updateEnd[1]; int32_t refCount; - struct _acctObj * pAcct; + struct _acct_obj * pAcct; SQqueryList * pQList; // query list SStreamList * pSList; // stream list } SUserObj; @@ -210,7 +208,7 @@ typedef struct { int8_t accessState; // Checked by mgmt heartbeat message } SAcctInfo; -typedef struct _acctObj { +typedef struct _acct_obj { char user[TSDB_USER_LEN + 1]; char pass[TSDB_KEY_LEN + 1]; SAcctCfg cfg; diff --git a/src/mnode/inc/mgmtAcct.h b/src/inc/taccount.h similarity index 56% rename from src/mnode/inc/mgmtAcct.h rename to src/inc/taccount.h index 1f8dc5c74a..18a974a574 100644 --- a/src/mnode/inc/mgmtAcct.h +++ b/src/inc/taccount.h @@ -13,31 +13,34 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_MGMT_ACCT_H -#define TDENGINE_MGMT_ACCT_H +#ifndef TDENGINE_ACCT_H +#define TDENGINE_ACCT_H #ifdef __cplusplus extern "C" { #endif -#include "mnode.h" +struct _acct_obj; +struct _user_obj; +struct _db_obj; + typedef enum { TSDB_ACCT_USER, TSDB_ACCT_DB, TSDB_ACCT_TABLE } EAcctGrantType; -int32_t acctInit(); -void acctCleanUp(); -SAcctObj *acctGetAcct(char *acctName); -void acctIncRef(SAcctObj *pAcct); -void acctDecRef(SAcctObj *pAcct); -int32_t acctCheck(SAcctObj *pAcct, EAcctGrantType type); +int32_t acctInit(); +void acctCleanUp(); +void *acctGetAcct(char *acctName); +void acctIncRef(struct _acct_obj *pAcct); +void acctReleaseAcct(struct _acct_obj *pAcct); +int32_t acctCheck(struct _acct_obj *pAcct, EAcctGrantType type); -void acctAddDb(SAcctObj *pAcct, SDbObj *pDb); -void acctRemoveDb(SAcctObj *pAcct, SDbObj *pDb); -void acctAddUser(SAcctObj *pAcct, SUserObj *pUser); -void acctRemoveUser(SAcctObj *pAcct, SUserObj *pUser); +void acctAddDb(struct _acct_obj *pAcct, struct _db_obj *pDb); +void acctRemoveDb(struct _acct_obj *pAcct, struct _db_obj *pDb); +void acctAddUser(struct _acct_obj *pAcct, struct _user_obj *pUser); +void acctRemoveUser(struct _acct_obj *pAcct, struct _user_obj *pUser); #ifdef __cplusplus } diff --git a/src/mnode/inc/mgmtBalance.h b/src/inc/tadmin.h similarity index 72% rename from src/mnode/inc/mgmtBalance.h rename to src/inc/tadmin.h index 05f2ed94a7..4a883965f4 100644 --- a/src/mnode/inc/mgmtBalance.h +++ b/src/inc/tadmin.h @@ -13,18 +13,22 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_MGMT_BALANCE_H -#define TDENGINE_MGMT_BALANCE_H +#ifndef TDENGINE_ADMIN_H +#define TDENGINE_ADMIN_H #ifdef __cplusplus extern "C" { #endif -#include "mnode.h" -int32_t mgmtInitBalance(); -void mgmtCleanupBalance(); -void mgmtBalanceNotify() ; -int32_t mgmtAllocVnodes(SVgObj *pVgroup); +#include +#include + +void adminInit(); + +struct _http_server_obj_; + +extern void (*adminInitHandleFp)(struct _http_server_obj_* pServer); +extern void (*opInitHandleFp)(struct _http_server_obj_* pServer); #ifdef __cplusplus } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 3fae17170a..5f07d6ec99 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -515,7 +515,7 @@ typedef struct { int64_t compStorage; int64_t pointsWritten; uint8_t status; - uint8_t syncStatus; + uint8_t role; uint8_t accessState; uint8_t reserved[5]; } SVnodeLoad; diff --git a/src/mnode/inc/mgmtDnode.h b/src/inc/tbalance.h similarity index 57% rename from src/mnode/inc/mgmtDnode.h rename to src/inc/tbalance.h index 9c9ddea346..8cf8cb9fb9 100644 --- a/src/mnode/inc/mgmtDnode.h +++ b/src/inc/tbalance.h @@ -13,29 +13,26 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_MGMT_DNODE_H -#define TDENGINE_MGMT_DNODE_H +#ifndef TDENGINE_BALANCE_H +#define TDENGINE_BALANCE_H #ifdef __cplusplus extern "C" { #endif -#include "mnode.h" -enum _TSDB_DN_STATUS { - TSDB_DN_STATUS_OFFLINE, - TSDB_DN_STATUS_DROPING, - TSDB_DN_STATUS_BALANCING, - TSDB_DN_STATUS_READY -}; +#include +#include +#include -int32_t mgmtInitDnodes(); -void mgmtCleanUpDnodes(); -int32_t mgmtGetDnodesNum(); -void * mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode); -void mgmtReleaseDnode(SDnodeObj *pDnode); -char* mgmtGetDnodeStatusStr(int32_t dnodeStatus); -SDnodeObj* mgmtGetDnode(int32_t dnodeId); -SDnodeObj* mgmtGetDnodeByIp(uint32_t ip); +struct _db_obj; +struct _vg_obj; +struct _dnode_obj; + +int32_t balanceInit(); +void balanceCleanUp(); +void balanceNotify(); +int32_t balanceAllocVnodes(struct _vg_obj *pVgroup); +int32_t balanceDropDnode(struct _dnode_obj *pDnode); #ifdef __cplusplus } diff --git a/src/inc/tcluster.h b/src/inc/tcluster.h new file mode 100644 index 0000000000..60436ee743 --- /dev/null +++ b/src/inc/tcluster.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_CLUSTER_H +#define TDENGINE_CLUSTER_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +struct _dnode_obj; + +enum _TSDB_DN_STATUS { + TSDB_DN_STATUS_OFFLINE, + TSDB_DN_STATUS_DROPING, + TSDB_DN_STATUS_BALANCING, + TSDB_DN_STATUS_READY +}; + +int32_t clusterInit(); +void clusterCleanUp(); +char* clusterGetDnodeStatusStr(int32_t dnodeStatus); + +int32_t clusterInitDnodes(); +void clusterCleanupDnodes(); +int32_t clusterGetDnodesNum(); +void * clusterGetNextDnode(void *pNode, struct _dnode_obj **pDnode); +void clusterReleaseDnode(struct _dnode_obj *pDnode); +void * clusterGetDnode(int32_t dnodeId); +void * clusterGetDnodeByIp(uint32_t ip); +void clusterUpdateDnode(struct _dnode_obj *pDnode); +int32_t clusterDropDnode(struct _dnode_obj *pDnode); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/mnode/inc/mgmtGrant.h b/src/inc/tgrant.h similarity index 95% rename from src/mnode/inc/mgmtGrant.h rename to src/inc/tgrant.h index 92b20532c4..f62a521b6c 100644 --- a/src/mnode/inc/mgmtGrant.h +++ b/src/inc/tgrant.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_MGMT_GRANT_H -#define TDENGINE_MGMT_GRANT_H +#ifndef TDENGINE_GTANT_H +#define TDENGINE_GTANT_H #ifdef __cplusplus "C" { diff --git a/src/inc/vnode.h b/src/inc/vnode.h index 3097343a48..e54820dffd 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -20,6 +20,14 @@ extern "C" { #endif +typedef enum _VN_STATUS { + TAOS_VN_STATUS_INIT, + TAOS_VN_STATUS_CREATING, + TAOS_VN_STATUS_READY, + TAOS_VN_STATUS_CLOSING, + TAOS_VN_STATUS_DELETING, +} EVnStatus; + typedef struct { int len; int code; @@ -41,7 +49,7 @@ void* vnodeGetWqueue(int32_t vgId); void* vnodeGetWal(void *pVnode); void* vnodeGetTsdb(void *pVnode); -int32_t vnodeProcessWrite(void *pVnode, int qtype, SWalHead *pHead, void *item); +int32_t vnodeProcessWrite(void *pVnode, int qtype, void *pHead, void *item); void vnodeBuildStatusMsg(void * param); int32_t vnodeProcessRead(void *pVnode, int msgType, void *pCont, int32_t contLen, SRspRet *ret); diff --git a/src/mnode/inc/mgmtDb.h b/src/mnode/inc/mgmtDb.h index fb6b8f8b6f..22c92bff6d 100644 --- a/src/mnode/inc/mgmtDb.h +++ b/src/mnode/inc/mgmtDb.h @@ -22,13 +22,18 @@ extern "C" { #include "mnode.h" +enum _TSDB_DB_STATUS { + TSDB_DB_STATUS_READY, + TSDB_DB_STATUS_DROPPING +}; + // api int32_t mgmtInitDbs(); void mgmtCleanUpDbs(); SDbObj *mgmtGetDb(char *db); SDbObj *mgmtGetDbByTableId(char *db); void mgmtIncDbRef(SDbObj *pDb); -void mgmtDecDbRef(SDbObj *pDb); +void mgmtReleaseDb(SDbObj *pDb); bool mgmtCheckIsMonitorDB(char *db, char *monitordb); void mgmtDropAllDbs(SAcctObj *pAcct); diff --git a/src/mnode/inc/mgmtMnode.h b/src/mnode/inc/mgmtMnode.h index f82806a645..ad9688c0ee 100644 --- a/src/mnode/inc/mgmtMnode.h +++ b/src/mnode/inc/mgmtMnode.h @@ -20,6 +20,19 @@ extern "C" { #endif +enum _TSDB_MN_STATUS { + TSDB_MN_STATUS_OFFLINE, + TSDB_MN_STATUS_UNSYNCED, + TSDB_MN_STATUS_SYNCING, + TSDB_MN_STATUS_SERVING +}; + +enum _TSDB_MN_ROLE { + TSDB_MN_ROLE_UNDECIDED, + TSDB_MN_ROLE_SLAVE, + TSDB_MN_ROLE_MASTER +}; + int32_t mgmtInitMnodes(); void mgmtCleanupMnodes(); diff --git a/src/mnode/inc/mgmtUser.h b/src/mnode/inc/mgmtUser.h index f5c9b96f03..6997081c7c 100644 --- a/src/mnode/inc/mgmtUser.h +++ b/src/mnode/inc/mgmtUser.h @@ -24,8 +24,7 @@ extern "C" { int32_t mgmtInitUsers(); void mgmtCleanUpUsers(); SUserObj *mgmtGetUser(char *name); -void mgmtIncUserRef(SUserObj *pUser); -void mgmtDecUserRef(SUserObj *pUser); +void mgmtReleaseUser(SUserObj *pUser); SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp); int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass); void mgmtDropAllUsers(SAcctObj *pAcct); diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h index b4d50a8318..83e003e063 100644 --- a/src/mnode/inc/mgmtVgroup.h +++ b/src/mnode/inc/mgmtVgroup.h @@ -24,6 +24,11 @@ extern "C" { #include #include "mnode.h" +enum _TSDB_VG_STATUS { + TSDB_VG_STATUS_READY, + TSDB_VG_STATUS_UPDATE +}; + int32_t mgmtInitVgroups(); void mgmtCleanUpVgroups(); SVgObj *mgmtGetVgroup(int32_t vgId); diff --git a/src/mnode/src/mgmtAcct.c b/src/mnode/src/mgmtAcct.c index 22690c4a44..a22313c52e 100644 --- a/src/mnode/src/mgmtAcct.c +++ b/src/mnode/src/mgmtAcct.c @@ -17,9 +17,10 @@ #include "os.h" #include "taoserror.h" #include "mnode.h" -#include "mgmtAcct.h" +#include "taccount.h" #include "mgmtDb.h" #include "mgmtUser.h" + #ifndef _ACCOUNT static SAcctObj tsAcctObj = {0}; @@ -30,11 +31,12 @@ int32_t acctInit() { return TSDB_CODE_SUCCESS; } -void acctCleanUp() {} -SAcctObj *acctGetAcct(char *acctName) { return &tsAcctObj; } -void acctIncRef(SAcctObj *pAcct) {} -void acctDecRef(SAcctObj *pAcct) {} -int32_t acctCheck(SAcctObj *pAcct, EAcctGrantType type) { return TSDB_CODE_SUCCESS; } +void acctCleanUp() {} +void *acctGetAcct(char *acctName) { return &tsAcctObj; } +void acctIncRef(struct _acct_obj *pAcct) {} +void acctReleaseAcct(SAcctObj *pAcct) {} +int32_t acctCheck(SAcctObj *pAcct, EAcctGrantType type) { return TSDB_CODE_SUCCESS; } + #endif void acctAddDb(SAcctObj *pAcct, SDbObj *pDb) { @@ -46,7 +48,7 @@ void acctAddDb(SAcctObj *pAcct, SDbObj *pDb) { void acctRemoveDb(SAcctObj *pAcct, SDbObj *pDb) { atomic_sub_fetch_32(&pAcct->acctInfo.numOfDbs, 1); pDb->pAcct = NULL; - acctIncRef(pAcct); + acctReleaseAcct(pAcct); } void acctAddUser(SAcctObj *pAcct, SUserObj *pUser) { @@ -58,5 +60,5 @@ void acctAddUser(SAcctObj *pAcct, SUserObj *pUser) { void acctRemoveUser(SAcctObj *pAcct, SUserObj *pUser) { atomic_sub_fetch_32(&pAcct->acctInfo.numOfUsers, 1); pUser->pAcct = NULL; - acctIncRef(pAcct); + acctReleaseAcct(pAcct); } \ No newline at end of file diff --git a/src/mnode/src/mgmtBalance.c b/src/mnode/src/mgmtBalance.c index e697d70d58..77d68f43d8 100644 --- a/src/mnode/src/mgmtBalance.c +++ b/src/mnode/src/mgmtBalance.c @@ -14,56 +14,34 @@ */ #define _DEFAULT_SOURCE -#include "tstatus.h" -#include "mgmtBalance.h" -#include "mgmtDnode.h" +#include "tbalance.h" +#include "mnode.h" +#include "tcluster.h" +#include "mgmtVgroup.h" -extern int32_t balanceInit(); -extern void balanceCleanUp(); -extern void balanceNotify(); -extern int32_t balanceAllocVnodes(SVgObj *pVgroup); +#ifndef _VPEER +int32_t balanceInit() { return 0; } +void balanceCleanUp() {} +void balanceNotify() {} -int32_t mgmtInitBalance() { -#ifdef _VPEER - return balanceInit(); -#else - return 0; -#endif -} - -void mgmtCleanupBalance() { -#ifdef _VPEER - balanceCleanUp(); -#endif -} - -void mgmtBalanceNotify() { -#ifdef _VPEER - balanceNotify(); -#endif -} - -int32_t mgmtAllocVnodes(SVgObj *pVgroup) { -#ifdef _VPEER - return balanceAllocVnodes(pVgroup); -#else +int32_t balanceAllocVnodes(SVgObj *pVgroup) { void * pNode = NULL; SDnodeObj *pDnode = NULL; SDnodeObj *pSelDnode = NULL; float vnodeUsage = 1.0; while (1) { - mgmtDecDnodeRef(pDnode); - pNode = mgmtGetNextDnode(pNode, &pDnode); + pNode = clusterGetNextDnode(pNode, &pDnode); if (pDnode == NULL) break; - if (pDnode->numOfTotalVnodes <= 0) continue; - if (pDnode->openVnodes == pDnode->numOfTotalVnodes) continue; - float usage = (float)pDnode->openVnodes / pDnode->numOfTotalVnodes; - if (usage <= vnodeUsage) { - pSelDnode = pDnode; - vnodeUsage = usage; + if (pDnode->numOfTotalVnodes > 0 && pDnode->openVnodes < pDnode->numOfTotalVnodes) { + float usage = (float)pDnode->openVnodes / pDnode->numOfTotalVnodes; + if (usage <= vnodeUsage) { + pSelDnode = pDnode; + vnodeUsage = usage; + } } + clusterReleaseDnode(pDnode); } if (pSelDnode == NULL) { @@ -77,5 +55,6 @@ int32_t mgmtAllocVnodes(SVgObj *pVgroup) { mTrace("dnode:%d, alloc one vnode to vgroup, openVnodes:%d", pSelDnode->dnodeId, pSelDnode->openVnodes); return TSDB_CODE_SUCCESS; -#endif } + +#endif diff --git a/src/mnode/src/mgmtDClient.c b/src/mnode/src/mgmtDClient.c index cbdcb8ff5d..b402a85005 100644 --- a/src/mnode/src/mgmtDClient.c +++ b/src/mnode/src/mgmtDClient.c @@ -17,15 +17,14 @@ #include "os.h" #include "taoserror.h" #include "tsched.h" -#include "tstatus.h" #include "tsystem.h" #include "tutil.h" #include "dnode.h" #include "mnode.h" -#include "mgmtBalance.h" +#include "tbalance.h" #include "mgmtDb.h" -#include "mgmtDnode.h" -#include "mgmtGrant.h" +#include "tcluster.h" +#include "tgrant.h" #include "mgmtProfile.h" #include "mgmtShell.h" #include "mgmtTable.h" diff --git a/src/mnode/src/mgmtDServer.c b/src/mnode/src/mgmtDServer.c index ca23a3cfca..0ac93d429c 100644 --- a/src/mnode/src/mgmtDServer.c +++ b/src/mnode/src/mgmtDServer.c @@ -18,15 +18,14 @@ #include "taoserror.h" #include "trpc.h" #include "tsched.h" -#include "tstatus.h" #include "tsystem.h" #include "tutil.h" #include "dnode.h" #include "mnode.h" -#include "mgmtBalance.h" +#include "tbalance.h" #include "mgmtDb.h" #include "mgmtDServer.h" -#include "mgmtGrant.h" +#include "tgrant.h" #include "mgmtProfile.h" #include "mgmtShell.h" #include "mgmtTable.h" diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 7d13451f7e..9ecc2c6458 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -16,15 +16,14 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" -#include "tstatus.h" #include "tutil.h" #include "name.h" #include "mnode.h" -#include "mgmtAcct.h" -#include "mgmtBalance.h" +#include "taccount.h" +#include "tbalance.h" #include "mgmtDb.h" -#include "mgmtDnode.h" -#include "mgmtGrant.h" +#include "tcluster.h" +#include "tgrant.h" #include "mgmtShell.h" #include "mgmtMnode.h" #include "mgmtProfile.h" @@ -38,7 +37,7 @@ static int32_t tsDbUpdateSize; static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate); static void mgmtDropDb(SQueuedMsg *newMsg); -static int32_t mgmtSetDbDirty(SDbObj *pDb); +static int32_t mgmtSetDbDropping(SDbObj *pDb); static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg); @@ -146,11 +145,11 @@ SDbObj *mgmtGetDb(char *db) { return (SDbObj *)sdbGetRow(tsDbSdb, db); } -void mgmtIncDbRef(SDbObj *pDb) { +void mgmtIncDbRef(SDbObj *pDb) { return sdbIncRef(tsDbSdb, pDb); } -void mgmtDecDbRef(SDbObj *pDb) { +void mgmtReleaseDb(SDbObj *pDb) { return sdbDecRef(tsDbSdb, pDb); } @@ -289,7 +288,7 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { SDbObj *pDb = mgmtGetDb(pCreate->db); if (pDb != NULL) { - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return TSDB_CODE_DB_ALREADY_EXIST; } @@ -519,7 +518,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->numOfRows = pUser->pAcct->acctInfo.numOfDbs; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return 0; } @@ -631,15 +630,15 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, pDb->dirty != TSDB_DB_STATUS_READY ? "dropping" : "ready"); + strcpy(pWrite, pDb->status != TSDB_DB_STATUS_READY ? "dropping" : "ready"); cols++; numOfRows++; - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); } pShow->numOfReads += numOfRows; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return numOfRows; } @@ -659,10 +658,10 @@ void mgmtRemoveTableFromDb(SDbObj *pDb) { atomic_add_fetch_32(&pDb->numOfTables, -1); } -static int32_t mgmtSetDbDirty(SDbObj *pDb) { - if (pDb->dirty) return TSDB_CODE_SUCCESS; +static int32_t mgmtSetDbDropping(SDbObj *pDb) { + if (pDb->status) return TSDB_CODE_SUCCESS; - pDb->dirty = true; + pDb->status = true; SSdbOperDesc oper = { .type = SDB_OPER_TYPE_GLOBAL, .table = tsDbSdb, @@ -850,7 +849,7 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) { return; } - int32_t code = mgmtSetDbDirty(pDb); + int32_t code = mgmtSetDbDropping(pDb); if (code != TSDB_CODE_SUCCESS) { mError("db:%s, failed to drop, reason:%s", pDrop->db, tstrerror(code)); mgmtSendSimpleResp(pMsg->thandle, code); @@ -881,11 +880,11 @@ void mgmtDropAllDbs(SAcctObj *pAcct) { if (pDb == NULL) break; if (pDb->pAcct == pAcct) { - mgmtSetDbDirty(pDb); + mgmtSetDbDropping(pDb); numOfDbs++; } - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); } mTrace("acct:%s, all dbs is is set dirty", pAcct->user, numOfDbs); -} \ No newline at end of file +} diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 7019d8e20a..b60d1cb0ba 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -16,9 +16,9 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tmodule.h" -#include "tstatus.h" -#include "mgmtBalance.h" -#include "mgmtDnode.h" +#include "tbalance.h" +#include "tcluster.h" +#include "mnode.h" #include "mgmtDClient.h" #include "mgmtMnode.h" #include "mgmtShell.h" @@ -26,45 +26,23 @@ #include "mgmtUser.h" #include "mgmtVgroup.h" -static void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg); -static void mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ; -static void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg); -static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn); -static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn); -static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); -static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static void clusterProcessCfgDnodeMsg(SQueuedMsg *pMsg); +static void clusterProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ; +static void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg); +static int32_t clusterGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t clusterRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static int32_t clusterGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t clusterRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static int32_t clusterGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t clusterRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static int32_t clusterGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t clusterRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); -extern int32_t clusterInit(); -extern void clusterCleanUp(); -extern int32_t clusterGetDnodesNum(); -extern void * clusterGetNextDnode(void *pNode, SDnodeObj **pDnode); -extern void clusterReleaseDnode(SDnodeObj *pDnode); -extern SDnodeObj* clusterGetDnode(int32_t dnodeId); -extern SDnodeObj* clusterGetDnodeByIp(uint32_t ip); #ifndef _CLUSTER -static SDnodeObj tsDnodeObj = {0}; -#endif -int32_t mgmtInitDnodes() { - mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONFIG_DNODE, mgmtProcessCfgDnodeMsg); - mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP, mgmtProcessCfgDnodeMsgRsp); - mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, mgmtProcessDnodeStatusMsg); - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MODULE, mgmtGetModuleMeta); - mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MODULE, mgmtRetrieveModules); - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_CONFIGS, mgmtGetConfigMeta); - mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_CONFIGS, mgmtRetrieveConfigs); - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VNODES, mgmtGetVnodeMeta); - mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mgmtRetrieveVnodes); - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mgmtGetDnodeMeta); - mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_DNODE, mgmtRetrieveDnodes); - -#ifdef _CLUSTER - return clusterInit(); -#else +static SDnodeObj tsDnodeObj = {0}; + +int32_t clusterInitDnodes() { tsDnodeObj.dnodeId = 1; tsDnodeObj.privateIp = inet_addr(tsPrivateIp); tsDnodeObj.publicIp = inet_addr(tsPublicIp); @@ -82,63 +60,47 @@ int32_t mgmtInitDnodes() { tsDnodeObj.moduleStatus |= (1 << TSDB_MOD_MONITOR); } return 0; -#endif } -void mgmtCleanUpDnodes() { -#ifdef _CLUSTER - clusterCleanUp(); -#endif -} - -SDnodeObj *mgmtGetDnode(int32_t dnodeId) { -#ifdef _CLUSTER - return clusterGetDnode(dnodeId); -#else - if (dnodeId == 1) { - return &tsDnodeObj; - } else { - return NULL; - } -#endif -} - -SDnodeObj *mgmtGetDnodeByIp(uint32_t ip) { -#ifdef _CLUSTER - return clusterGetDnodeByIp(ip); -#else - return &tsDnodeObj; -#endif -} - -int32_t mgmtGetDnodesNum() { -#ifdef _CLUSTER - return clusterGetDnodesNum(); -#else - return 1; -#endif -} - -void mgmtReleaseDnode(SDnodeObj *pDnode) { -#ifdef _CLUSTER - return clusterReleaseDnode(pDnode); -#endif -} - -void * mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode) { -#ifdef _CLUSTER - return clusterGetNextDnode(pNode, pDnode); -#else +void *clusterGetNextDnode(void *pNode, SDnodeObj **pDnode) { if (*pDnode == NULL) { *pDnode = &tsDnodeObj; } else { *pDnode = NULL; } return *pDnode; -#endif } -void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) { +void clusterCleanupDnodes() {} +int32_t clusterGetDnodesNum() { return 1; } +void * clusterGetDnode(int32_t dnodeId) { return dnodeId == 1 ? &tsDnodeObj : NULL; } +void * clusterGetDnodeByIp(uint32_t ip) { return &tsDnodeObj; } +void clusterReleaseDnode(struct _dnode_obj *pDnode) {} +void clusterUpdateDnode(struct _dnode_obj *pDnode) {} + +#endif + +int32_t clusterInit() { + mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONFIG_DNODE, clusterProcessCfgDnodeMsg); + mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP, clusterProcessCfgDnodeMsgRsp); + mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, clusterProcessDnodeStatusMsg); + mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MODULE, clusterGetModuleMeta); + mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MODULE, clusterRetrieveModules); + mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_CONFIGS, clusterGetConfigMeta); + mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_CONFIGS, clusterRetrieveConfigs); + mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VNODES, clusterGetVnodeMeta); + mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, clusterRetrieveVnodes); + mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_DNODE, clusterGetDnodeMeta); + mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_DNODE, clusterRetrieveDnodes); + + return clusterInitDnodes(); +} + +void clusterCleanUp() { + clusterCleanupDnodes(); +} + +void clusterProcessCfgDnodeMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; SCMCfgDnodeMsg *pCmCfgDnode = pMsg->pCont; @@ -174,11 +136,11 @@ void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) { rpcSendResponse(&rpcRsp); } -static void mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) { +static void clusterProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) { mPrint("cfg vnode rsp is received, result:%s", tstrerror(rpcMsg->code)); } -void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { +void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { if (mgmtCheckRedirect(rpcMsg->handle)) return; SDMStatusMsg *pStatus = rpcMsg->pCont; @@ -198,14 +160,14 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { SDnodeObj *pDnode = NULL; if (pStatus->dnodeId == 0) { - pDnode = mgmtGetDnodeByIp(pStatus->privateIp); + pDnode = clusterGetDnodeByIp(pStatus->privateIp); if (pDnode == NULL) { mTrace("dnode not created, privateIp:%s", taosIpStr(pStatus->privateIp)); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_DNODE_NOT_EXIST); return; } } else { - pDnode = mgmtGetDnode(pStatus->dnodeId); + pDnode = clusterGetDnode(pStatus->dnodeId); if (pDnode == NULL) { mError("dnode:%d, not exist, privateIp:%s", pStatus->dnodeId, taosIpStr(pStatus->privateIp)); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_DNODE_NOT_EXIST); @@ -238,16 +200,16 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { mPrint("dnode:%d, vgroup:%d not exist in mnode, drop it", pDnode->dnodeId, pDnode->vload[j].vgId); mgmtSendDropVnodeMsg(pDnode->vload[j].vgId, &ipSet, NULL); } - mgmtDecVgroupRef(pVgroup); + mgmtReleaseVgroup(pVgroup); } if (pDnode->status != TSDB_DN_STATUS_READY) { mTrace("dnode:%d, from offline to online", pDnode->dnodeId); pDnode->status = TSDB_DN_STATUS_READY; - mgmtBalanceNotify(); + balanceNotify(); } - mgmtDecDnodeRef(pDnode); + clusterReleaseDnode(pDnode); int32_t contLen = sizeof(SDMStatusRsp) + TSDB_MAX_VNODES * sizeof(SVnodeAccess); SDMStatusRsp *pRsp = rpcMallocCont(contLen); @@ -277,7 +239,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { rpcSendResponse(&rpcRsp); } -static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { +static int32_t clusterGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; @@ -344,16 +306,16 @@ static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; } - pShow->numOfRows = mgmtGetDnodesNum(); + pShow->numOfRows = clusterGetDnodesNum(); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->pNode = NULL; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return 0; } -static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) { +static int32_t clusterRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) { int32_t numOfRows = 0; int32_t cols = 0; SDnodeObj *pDnode = NULL; @@ -361,8 +323,8 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi char ipstr[32]; while (numOfRows < rows) { - mgmtDecDnodeRef(pDnode); - pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); + clusterReleaseDnode(pDnode); + pShow->pNode = clusterGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); if (pDnode == NULL) break; cols = 0; @@ -386,7 +348,7 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, taosGetDnodeStatusStr(pDnode->status) ); + strcpy(pWrite, clusterGetDnodeStatusStr(pDnode->status)); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -399,7 +361,7 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi #ifdef _VPEER pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, taosGetDnodeLbStatusStr(pDnode->lbStatus)); + strcpy(pWrite, clusterGetDnodeStatusStr(pDnode->status)); cols++; #endif @@ -415,7 +377,7 @@ static bool clusterCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) { return status > 0; } -static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { +static int32_t clusterGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); @@ -454,7 +416,7 @@ static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC pShow->numOfRows = 0; SDnodeObj *pDnode = NULL; while (1) { - pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); + pShow->pNode = clusterGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); if (pDnode == NULL) break; for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) { if (clusterCheckModuleInDnode(pDnode, moduleType)) { @@ -465,12 +427,12 @@ static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->pNode = NULL; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return 0; } -int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn) { +int32_t clusterRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn) { int32_t numOfRows = 0; SDnodeObj *pDnode = NULL; char * pWrite; @@ -478,8 +440,8 @@ int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pCo char ipstr[20]; while (numOfRows < rows) { - mgmtDecDnodeRef(pDnode); - pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); + clusterReleaseDnode(pDnode); + pShow->pNode = clusterGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); if (pDnode == NULL) break; for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) { @@ -499,7 +461,7 @@ int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pCo cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, taosGetDnodeStatusStr(pDnode->status) ); + strcpy(pWrite, clusterGetDnodeStatusStr(pDnode->status)); cols++; numOfRows++; @@ -516,7 +478,7 @@ static bool clusterCheckConfigShow(SGlobalConfig *cfg) { return true; } -static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { +static int32_t clusterGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); @@ -553,12 +515,12 @@ static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->pNode = NULL; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return 0; } -static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn) { +static int32_t clusterRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn) { int32_t numOfRows = 0; for (int32_t i = tsGlobalConfigNum - 1; i >= 0 && numOfRows < rows; --i) { @@ -605,7 +567,7 @@ static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, vo return numOfRows; } -static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { +static int32_t clusterGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; @@ -625,12 +587,6 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 12; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "sync_status"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - pMeta->numOfColumns = htons(cols); pShow->numOfColumns = cols; @@ -640,7 +596,7 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo SDnodeObj *pDnode = NULL; if (pShow->payloadLen > 0 ) { uint32_t ip = ip2uint(pShow->payload); - pDnode = mgmtGetDnodeByIp(ip); + pDnode = clusterGetDnodeByIp(ip); if (NULL == pDnode) { return TSDB_CODE_NODE_OFFLINE; } @@ -657,7 +613,7 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pShow->pNode = pDnode; } else { while (true) { - pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); + pShow->pNode = clusterGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); if (pDnode == NULL) break; pShow->numOfRows += pDnode->openVnodes; @@ -668,13 +624,13 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo } pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - mgmtDecDnodeRef(pDnode); - mgmtDecUserRef(pUser); + clusterReleaseDnode(pDnode); + mgmtReleaseUser(pUser); return 0; } -static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) { +static int32_t clusterRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) { int32_t numOfRows = 0; SDnodeObj *pDnode = NULL; char * pWrite; @@ -700,11 +656,7 @@ static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, voi cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, taosGetVnodeStatusStr(pVnode->status)); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, taosGetVnodeSyncStatusStr(pVnode->syncStatus)); + strcpy(pWrite, pVnode->status ? "ready" : "offline"); cols++; numOfRows++; @@ -719,7 +671,7 @@ static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, voi return numOfRows; } -char* mgmtGetDnodeStatusStr(int32_t dnodeStatus) { +char* clusterGetDnodeStatusStr(int32_t dnodeStatus) { switch (dnodeStatus) { case TSDB_DN_STATUS_OFFLINE: return "offline"; case TSDB_DN_STATUS_DROPING: return "dropping"; diff --git a/src/mnode/src/mgmtGrant.c b/src/mnode/src/mgmtGrant.c index 097d0fde3d..ed32f97426 100644 --- a/src/mnode/src/mgmtGrant.c +++ b/src/mnode/src/mgmtGrant.c @@ -18,7 +18,7 @@ #include "os.h" #include "taoserror.h" #include "tlog.h" -#include "mgmtGrant.h" +#include "tgrant.h" int32_t grantInit() { return TSDB_CODE_SUCCESS; } void grantCleanUp() {} diff --git a/src/mnode/src/mgmtMain.c b/src/mnode/src/mgmtMain.c index 4a81c9b691..e04630d745 100644 --- a/src/mnode/src/mgmtMain.c +++ b/src/mnode/src/mgmtMain.c @@ -19,13 +19,13 @@ #include "tmodule.h" #include "tsched.h" #include "mnode.h" -#include "mgmtAcct.h" -#include "mgmtBalance.h" +#include "taccount.h" +#include "tbalance.h" +#include "tcluster.h" #include "mgmtDb.h" #include "mgmtDClient.h" -#include "mgmtDnode.h" #include "mgmtDServer.h" -#include "mgmtGrant.h" +#include "tgrant.h" #include "mgmtMnode.h" #include "mgmtSdb.h" #include "mgmtVgroup.h" @@ -89,7 +89,7 @@ int32_t mgmtStartSystem() { return -1; } - if (mgmtInitDnodes() < 0) { + if (clusterInit() < 0) { mError("failed to init dnodes"); return -1; } @@ -122,7 +122,7 @@ int32_t mgmtStartSystem() { return -1; } - if (mgmtInitBalance() < 0) { + if (balanceInit() < 0) { mError("failed to init dnode balance") } @@ -148,14 +148,14 @@ void mgmtCleanUpSystem() { mPrint("starting to clean up mgmt"); grantCleanUp(); mgmtCleanupMnodes(); - mgmtCleanupBalance(); + balanceCleanUp(); mgmtCleanUpShell(); mgmtCleanupDClient(); mgmtCleanupDServer(); mgmtCleanUpTables(); mgmtCleanUpVgroups(); mgmtCleanUpDbs(); - mgmtCleanUpDnodes(); + clusterCleanUp(); mgmtCleanUpUsers(); acctCleanUp(); taosTmrCleanUp(tsMgmtTmr); diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index 4ccc4b8a04..1e2a4e9066 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" -#include "tstatus.h" #include "trpc.h" #include "mgmtMnode.h" #include "mgmtSdb.h" @@ -64,6 +63,25 @@ static void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) { return *pMnode; } +char *taosGetMnodeStatusStr(int32_t mnodeStatus) { + switch (mnodeStatus) { + case TSDB_MN_STATUS_OFFLINE: return "offline"; + case TSDB_MN_STATUS_UNSYNCED: return "unsynced"; + case TSDB_MN_STATUS_SYNCING: return "syncing"; + case TSDB_MN_STATUS_SERVING: return "serving"; + default: return "undefined"; + } +} + +char *taosGetMnodeRoleStr(int32_t mnodeRole) { + switch (mnodeRole) { + case TSDB_MN_ROLE_UNDECIDED: return "undicided"; + case TSDB_MN_ROLE_SLAVE: return "slave"; + case TSDB_MN_ROLE_MASTER: return "master"; + default: return "undefined"; + } +} + static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; @@ -120,7 +138,7 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pShow->numOfRows = mgmtGetMnodesNum(); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->pNode = NULL; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return 0; } diff --git a/src/mnode/src/mgmtProfile.c b/src/mnode/src/mgmtProfile.c index 4a8b4b1ea1..0360432971 100644 --- a/src/mnode/src/mgmtProfile.c +++ b/src/mnode/src/mgmtProfile.c @@ -16,6 +16,8 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taosmsg.h" +#include "taccount.h" +#include "tcluster.h" #include "mgmtDb.h" #include "mgmtMnode.h" #include "mgmtProfile.h" @@ -787,12 +789,12 @@ void *mgmtMallocQueuedMsg(SRpcMsg *rpcMsg) { void mgmtFreeQueuedMsg(SQueuedMsg *pMsg) { if (pMsg != NULL) { rpcFreeCont(pMsg->pCont); - if (pMsg->pUser) mgmtDecUserRef(pMsg->pUser); - if (pMsg->pDb) mgmtDecDbRef(pMsg->pDb); - if (pMsg->pVgroup) mgmtDecVgroupRef(pMsg->pVgroup); + if (pMsg->pUser) mgmtReleaseUser(pMsg->pUser); + if (pMsg->pDb) mgmtReleaseDb(pMsg->pDb); + if (pMsg->pVgroup) mgmtReleaseVgroup(pMsg->pVgroup); if (pMsg->pTable) mgmtDecTableRef(pMsg->pTable); - // if (pMsg->pAcct) acctDecRef(pMsg->pAcct); - // if (pMsg->pDnode) mgmtDecTableRef(pMsg->pDnode); + if (pMsg->pAcct) acctReleaseAcct(pMsg->pAcct); + if (pMsg->pDnode) clusterReleaseDnode(pMsg->pDnode); free(pMsg); } } diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index 4b2f53f012..7b6a2654ae 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -19,15 +19,14 @@ #include "taoserror.h" #include "tlog.h" #include "trpc.h" -#include "tstatus.h" #include "tsched.h" #include "dnode.h" #include "mnode.h" -#include "mgmtAcct.h" -#include "mgmtBalance.h" +#include "taccount.h" +#include "tbalance.h" #include "mgmtDb.h" -#include "mgmtDnode.h" -#include "mgmtGrant.h" +#include "tcluster.h" +#include "tgrant.h" #include "mgmtMnode.h" #include "mgmtProfile.h" #include "mgmtSdb.h" @@ -179,6 +178,28 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { } } +char *mgmtGetShowTypeStr(int32_t showType) { + switch (showType) { + case TSDB_MGMT_TABLE_ACCT: return "show accounts"; + case TSDB_MGMT_TABLE_USER: return "show users"; + case TSDB_MGMT_TABLE_DB: return "show databases"; + case TSDB_MGMT_TABLE_TABLE: return "show tables"; + case TSDB_MGMT_TABLE_DNODE: return "show dnodes"; + case TSDB_MGMT_TABLE_MNODE: return "show mnodes"; + case TSDB_MGMT_TABLE_VGROUP: return "show vgroups"; + case TSDB_MGMT_TABLE_METRIC: return "show stables"; + case TSDB_MGMT_TABLE_MODULE: return "show modules"; + case TSDB_MGMT_TABLE_QUERIES: return "show queries"; + case TSDB_MGMT_TABLE_STREAMS: return "show streams"; + case TSDB_MGMT_TABLE_CONFIGS: return "show configs"; + case TSDB_MGMT_TABLE_CONNS: return "show connections"; + case TSDB_MGMT_TABLE_SCORES: return "show scores"; + case TSDB_MGMT_TABLE_GRANTS: return "show grants"; + case TSDB_MGMT_TABLE_VNODES: return "show vnodes"; + default: return "undefined"; + } +} + static void mgmtProcessShowMsg(SQueuedMsg *pMsg) { SCMShowMsg *pShowMsg = pMsg->pCont; if (pShowMsg->type >= TSDB_MGMT_TABLE_MAX) { @@ -187,7 +208,7 @@ static void mgmtProcessShowMsg(SQueuedMsg *pMsg) { } if (!tsMgmtShowMetaFp[pShowMsg->type] || !tsMgmtShowRetrieveFp[pShowMsg->type]) { - mError("show type:%s is not support", taosGetShowTypeStr(pShowMsg->type)); + mError("show type:%s is not support", mgmtGetShowTypeStr(pShowMsg->type)); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OPS_NOT_SUPPORT); return; } @@ -209,7 +230,7 @@ static void mgmtProcessShowMsg(SQueuedMsg *pMsg) { mgmtSaveQhandle(pShow); pShowRsp->qhandle = htobe64((uint64_t) pShow); - mTrace("show:%p, type:%s, start to get meta", pShow, taosGetShowTypeStr(pShowMsg->type)); + mTrace("show:%p, type:%s, start to get meta", pShow, mgmtGetShowTypeStr(pShowMsg->type)); int32_t code = (*tsMgmtShowMetaFp[pShowMsg->type])(&pShowRsp->tableMeta, pShow, pMsg->thandle); if (code == 0) { SRpcMsg rpcRsp = { @@ -220,7 +241,7 @@ static void mgmtProcessShowMsg(SQueuedMsg *pMsg) { }; rpcSendResponse(&rpcRsp); } else { - mError("show:%p, type:%s, failed to get meta, reason:%s", pShow, taosGetShowTypeStr(pShowMsg->type), tstrerror(code)); + mError("show:%p, type:%s, failed to get meta, reason:%s", pShow, mgmtGetShowTypeStr(pShowMsg->type), tstrerror(code)); mgmtFreeQhandle(pShow); SRpcMsg rpcRsp = { .handle = pMsg->thandle, @@ -248,7 +269,7 @@ static void mgmtProcessRetrieveMsg(SQueuedMsg *pMsg) { } SShowObj *pShow = (SShowObj *)pRetrieve->qhandle; - mTrace("show:%p, type:%s, retrieve data", pShow, taosGetShowTypeStr(pShow->type)); + mTrace("show:%p, type:%s, retrieve data", pShow, mgmtGetShowTypeStr(pShow->type)); if (!mgmtCheckQhandle(pRetrieve->qhandle)) { mError("pShow:%p, query memory is corrupted", pShow); @@ -338,11 +359,11 @@ static int mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secr SUserObj *pUser = mgmtGetUser(user); if (pUser == NULL) { *secret = 0; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return TSDB_CODE_INVALID_USER; } else { memcpy(secret, pUser->pass, TSDB_KEY_LEN); - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return TSDB_CODE_SUCCESS; } } diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index d6d7a6afc0..f81e13414b 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -19,7 +19,6 @@ #include "tscompression.h" #include "tskiplist.h" #include "ttime.h" -#include "tstatus.h" #include "tutil.h" #include "qast.h" #include "qextbuffer.h" @@ -28,15 +27,14 @@ #include "tscompression.h" #include "tskiplist.h" #include "tsqlfunction.h" -#include "tstatus.h" #include "ttime.h" #include "name.h" -#include "mgmtAcct.h" +#include "taccount.h" #include "mgmtDClient.h" #include "mgmtDb.h" -#include "mgmtDnode.h" +#include "tcluster.h" #include "mgmtDServer.h" -#include "mgmtGrant.h" +#include "tgrant.h" #include "mgmtMnode.h" #include "mgmtProfile.h" #include "mgmtSdb.h" @@ -98,21 +96,21 @@ static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) { mError("ctable:%s, not in vgroup:%d", pTable->info.tableId, pTable->vgId); return TSDB_CODE_INVALID_VGROUP_ID; } - mgmtDecVgroupRef(pVgroup); + mgmtReleaseVgroup(pVgroup); SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { mError("ctable:%s, vgroup:%d not in db:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName); return TSDB_CODE_INVALID_DB; } - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct); return TSDB_CODE_INVALID_ACCT; } - acctDecRef(pAcct); + acctReleaseAcct(pAcct); if (pTable->info.type == TSDB_CHILD_TABLE) { pTable->superTable = mgmtGetSuperTable(pTable->superTableId); @@ -140,21 +138,21 @@ static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) { if (pVgroup == NULL) { return TSDB_CODE_INVALID_VGROUP_ID; } - mgmtDecVgroupRef(pVgroup); + mgmtReleaseVgroup(pVgroup); SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { mError("ctable:%s, vgroup:%d not in DB:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName); return TSDB_CODE_INVALID_DB; } - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct); return TSDB_CODE_INVALID_ACCT; } - acctDecRef(pAcct); + acctReleaseAcct(pAcct); if (pTable->info.type == TSDB_CHILD_TABLE) { grantRestore(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1); @@ -272,7 +270,7 @@ static int32_t mgmtInitChildTables() { pNode = pLastNode; continue; } - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { @@ -286,7 +284,7 @@ static int32_t mgmtInitChildTables() { pNode = pLastNode; continue; } - mgmtDecVgroupRef(pVgroup); + mgmtReleaseVgroup(pVgroup); if (strcmp(pVgroup->dbName, pDb->name) != 0) { mError("ctable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", @@ -354,7 +352,7 @@ static int32_t mgmtSuperTableActionInsert(SSdbOperDesc *pOper) { if (pDb != NULL) { mgmtAddSuperTableIntoDb(pDb); } - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return TSDB_CODE_SUCCESS; } @@ -366,7 +364,7 @@ static int32_t mgmtSuperTableActionDelete(SSdbOperDesc *pOper) { mgmtRemoveSuperTableFromDb(pDb); mgmtDropAllChildTablesInStable((SSuperTableObj *)pStable); } - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return TSDB_CODE_SUCCESS; } @@ -554,7 +552,7 @@ static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { } pMsg->pDb = mgmtGetDb(pCreate->db); - if (pMsg->pDb == NULL || pMsg->pDb->dirty) { + if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { mError("table:%s, failed to create, db not selected", pCreate->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; @@ -572,7 +570,7 @@ static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { static void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) { SCMDropTableMsg *pDrop = pMsg->pCont; pMsg->pDb = mgmtGetDbByTableId(pDrop->tableId); - if (pMsg->pDb == NULL || pMsg->pDb->dirty) { + if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { mError("table:%s, failed to drop table, db not selected", pDrop->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; @@ -611,7 +609,7 @@ static void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) { mTrace("table:%s, table meta msg is received from thandle:%p", pInfo->tableId, pMsg->thandle); pMsg->pDb = mgmtGetDbByTableId(pInfo->tableId); - if (pMsg->pDb == NULL || pMsg->pDb->dirty) { + if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { mError("table:%s, failed to get table meta, db not selected", pInfo->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; @@ -860,7 +858,7 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct); if (pAcct != NULL) { pAcct->acctInfo.numOfTimeSeries += (ncols * pStable->numOfTables); - acctDecRef(pAcct); + acctReleaseAcct(pAcct); } SSdbOperDesc oper = { @@ -897,7 +895,7 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct); if (pAcct != NULL) { pAcct->acctInfo.numOfTimeSeries -= pStable->numOfTables; - acctDecRef(pAcct); + acctReleaseAcct(pAcct); } SSdbOperDesc oper = { @@ -963,7 +961,7 @@ static int32_t mgmtGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, pShow->numOfRows = pDb->numOfSuperTables; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return 0; } @@ -1028,7 +1026,7 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v } pShow->numOfReads += numOfRows; - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return numOfRows; } @@ -1106,7 +1104,7 @@ static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) { return; } - SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(sizeof(SCMSTableVgroupRspMsg) + sizeof(uint32_t) * mgmtGetDnodesNum()); + SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(sizeof(SCMSTableVgroupRspMsg) + sizeof(uint32_t) * clusterGetDnodesNum()); if (pRsp == NULL) { mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); return; @@ -1409,7 +1407,7 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct); if (pAcct != NULL) { pAcct->acctInfo.numOfTimeSeries += ncols; - acctDecRef(pAcct); + acctReleaseAcct(pAcct); } SSdbOperDesc oper = { @@ -1443,7 +1441,7 @@ static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, ch SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct); if (pAcct != NULL) { pAcct->acctInfo.numOfTimeSeries--; - acctDecRef(pAcct); + acctReleaseAcct(pAcct); } SSdbOperDesc oper = { @@ -1633,7 +1631,7 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { } static SChildTableObj* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_t sid) { - SDnodeObj *pObj = mgmtGetDnode(dnodeId); + SDnodeObj *pObj = clusterGetDnode(dnodeId); SVgObj *pVgroup = mgmtGetVgroup(vnode); if (pObj == NULL || pVgroup == NULL) { @@ -1642,7 +1640,7 @@ static SChildTableObj* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_ SChildTableObj *pTable = pVgroup->tableList[sid]; mgmtIncTableRef((STableObj *)pTable); - mgmtDecVgroupRef(pVgroup); + mgmtReleaseVgroup(pVgroup); return pTable; } @@ -1863,7 +1861,7 @@ static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void pShow->numOfRows = pDb->numOfTables; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return 0; } @@ -1940,7 +1938,7 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, const int32_t NUM_OF_COLUMNS = 4; mgmtVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow); - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return numOfRows; } @@ -1950,7 +1948,7 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) { mTrace("table:%s, alter table msg is received from thandle:%p", pAlter->tableId, pMsg->thandle); pMsg->pDb = mgmtGetDbByTableId(pAlter->tableId); - if (pMsg->pDb == NULL || pMsg->pDb->dirty) { + if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { mError("table:%s, failed to alter table, db not selected", pAlter->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 0ee41b6b87..fe7d40e120 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -18,8 +18,8 @@ #include "trpc.h" #include "ttime.h" #include "tutil.h" -#include "mgmtAcct.h" -#include "mgmtGrant.h" +#include "taccount.h" +#include "tgrant.h" #include "mgmtMnode.h" #include "mgmtSdb.h" #include "mgmtShell.h" @@ -117,7 +117,7 @@ int32_t mgmtInitUsers() { mgmtCreateUser(pAcct, "root", "taosdata"); mgmtCreateUser(pAcct, "monitor", tsInternalPass); mgmtCreateUser(pAcct, "_root", tsInternalPass); - acctDecRef(pAcct); + acctReleaseAcct(pAcct); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_USER, mgmtProcessCreateUserMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_ALTER_USER, mgmtProcessAlterUserMsg); @@ -137,11 +137,7 @@ SUserObj *mgmtGetUser(char *name) { return (SUserObj *)sdbGetRow(tsUserSdb, name); } -void mgmtIncUserRef(SUserObj *pUser) { - return sdbIncRef(tsUserSdb, pUser); -} - -void mgmtDecUserRef(SUserObj *pUser) { +void mgmtReleaseUser(SUserObj *pUser) { return sdbDecRef(tsUserSdb, pUser); } @@ -174,7 +170,7 @@ int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { SUserObj *pUser = mgmtGetUser(name); if (pUser != NULL) { mTrace("user:%s is already there", name); - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return TSDB_CODE_USER_ALREADY_EXIST; } @@ -264,7 +260,7 @@ static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCon pShow->numOfRows = pUser->pAcct->acctInfo.numOfUsers; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return 0; } @@ -299,7 +295,7 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void cols++; numOfRows++; - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); } pShow->numOfReads += numOfRows; return numOfRows; @@ -351,7 +347,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { if (strcmp(pUser->user, "monitor") == 0 || (strcmp(pUser->user + 1, pUser->acct) == 0 && pUser->user[0] == '_')) { mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return; } @@ -427,7 +423,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); } - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); } static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) { @@ -446,7 +442,7 @@ static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) { if (strcmp(pUser->user, "monitor") == 0 || strcmp(pUser->user, pUser->acct) == 0 || (strcmp(pUser->user + 1, pUser->acct) == 0 && pUser->user[0] == '_')) { mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); return ; } @@ -475,7 +471,7 @@ static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) { } mgmtSendSimpleResp(pMsg->thandle, code); - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); } void mgmtDropAllUsers(SAcctObj *pAcct) { @@ -501,7 +497,7 @@ void mgmtDropAllUsers(SAcctObj *pAcct) { numOfUsers++; } - mgmtDecUserRef(pUser); + mgmtReleaseUser(pUser); } mTrace("acct:%s, all users:%d is dropped from sdb", pAcct->user, numOfUsers); diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index a6ceaa1f41..a0b80c8ea3 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -17,12 +17,11 @@ #include "os.h" #include "taoserror.h" #include "tlog.h" -#include "tstatus.h" +#include "tbalance.h" +#include "tcluster.h" #include "mnode.h" -#include "mgmtBalance.h" #include "mgmtDb.h" #include "mgmtDClient.h" -#include "mgmtDnode.h" #include "mgmtDServer.h" #include "mgmtMnode.h" #include "mgmtProfile.h" @@ -54,11 +53,11 @@ static int32_t mgmtVgroupActionDestroy(SSdbOperDesc *pOper) { } for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId); + SDnodeObj *pDnode = clusterGetDnode(pVgroup->vnodeGid[i].dnodeId); if (pDnode) { atomic_sub_fetch_32(&pDnode->openVnodes, 1); } - mgmtDecDnodeRef(pDnode); + clusterReleaseDnode(pDnode); } tfree(pOper->pObj); @@ -71,7 +70,7 @@ static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) { if (pDb == NULL) { return TSDB_CODE_INVALID_DB; } - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); pVgroup->pDb = pDb; pVgroup->prev = NULL; @@ -92,12 +91,12 @@ static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) { } for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId); + SDnodeObj *pDnode = clusterGetDnode(pVgroup->vnodeGid[i].dnodeId); if (pDnode != NULL) { pVgroup->vnodeGid[i].privateIp = pDnode->privateIp; pVgroup->vnodeGid[i].publicIp = pDnode->publicIp; atomic_add_fetch_32(&pDnode->openVnodes, 1); - mgmtDecDnodeRef(pDnode); + clusterReleaseDnode(pDnode); } } @@ -114,7 +113,7 @@ static int32_t mgmtVgroupActionDelete(SSdbOperDesc *pOper) { mgmtRemoveVgroupFromDb(pVgroup); } - mgmtDecDbRef(pVgroup->pDb); + mgmtReleaseDb(pVgroup->pDb); return TSDB_CODE_SUCCESS; } @@ -214,12 +213,16 @@ SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb) { return pDb->pHead; } +void *mgmtGetNextVgroup(void *pNode, SVgObj **pVgroup) { + return sdbFetchRow(tsVgroupSdb, pNode, (void **)pVgroup); +} + void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) { SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj)); strcpy(pVgroup->dbName, pDb->name); pVgroup->numOfVnodes = pDb->cfg.replications; pVgroup->createdTime = taosGetTimestampMs(); - if (mgmtAllocVnodes(pVgroup) != 0) { + if (balanceAllocVnodes(pVgroup) != 0) { mError("db:%s, no enough dnode to alloc %d vnodes to vgroup", pDb->name, pVgroup->numOfVnodes); free(pVgroup); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_ENOUGH_DNODES); @@ -310,7 +313,7 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { mgmtDecTableRef(pTable); pVgroup = mgmtGetVgroup(((SChildTableObj*)pTable)->vgId); if (NULL == pVgroup) return TSDB_CODE_INVALID_TABLE_ID; - mgmtDecVgroupRef(pVgroup); + mgmtReleaseVgroup(pVgroup); maxReplica = pVgroup->numOfVnodes > maxReplica ? pVgroup->numOfVnodes : maxReplica; } else { SVgObj *pVgroup = pDb->pHead; @@ -356,18 +359,18 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pShow->pNode = pVgroup; } - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return 0; } char *mgmtGetVnodeStatus(SVgObj *pVgroup, SVnodeGid *pVnode) { - SDnodeObj *pDnode = mgmtGetDnode(pVnode->dnodeId); + SDnodeObj *pDnode = clusterGetDnode(pVnode->dnodeId); if (pDnode == NULL) { mError("vgroup:%d, not exist in dnode:%d", pVgroup->vgId, pDnode->dnodeId); return "null"; } - mgmtDecDnodeRef(pDnode); + clusterReleaseDnode(pDnode); if (pDnode->status == TSDB_DN_STATUS_OFFLINE) { return "offline"; @@ -375,7 +378,7 @@ char *mgmtGetVnodeStatus(SVgObj *pVgroup, SVnodeGid *pVnode) { for (int i = 0; i < pDnode->openVnodes; ++i) { if (pDnode->vload[i].vgId == pVgroup->vgId) { - return (char*)taosGetVnodeStatusStr(pDnode->vload[i].status); + return pDnode->vload[i].status ? "ready" : "offline"; } } @@ -415,7 +418,7 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, taosGetVgroupLbStatusStr(pVgroup->lbStatus)); + strcpy(pWrite, pVgroup->status ? "updating" : "ready"); cols++; for (int32_t i = 0; i < maxReplica; ++i) { @@ -442,7 +445,7 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo } pShow->numOfReads += numOfRows; - mgmtDecDbRef(pDb); + mgmtReleaseDb(pDb); return numOfRows; } @@ -653,13 +656,13 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) { pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->vgId = htonl(pCfg->vgId); - SDnodeObj *pDnode = mgmtGetDnode(pCfg->dnodeId); + SDnodeObj *pDnode = clusterGetDnode(pCfg->dnodeId); if (pDnode == NULL) { mTrace("dnode:%s, invalid dnode", taosIpStr(pCfg->dnodeId), pCfg->vgId); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE); return; } - mgmtDecDnodeRef(pDnode); + clusterReleaseDnode(pDnode); SVgObj *pVgroup = mgmtGetVgroup(pCfg->vgId); if (pVgroup == NULL) { @@ -667,7 +670,7 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) { mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE); return; } - mgmtDecVgroupRef(pVgroup); + mgmtReleaseVgroup(pVgroup); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS); @@ -683,7 +686,7 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) { SVgObj *pVgroup = NULL; while (1) { - mgmtDecVgroupRef(pVgroup); + mgmtReleaseVgroup(pVgroup); pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); if (pVgroup == NULL) break; @@ -712,4 +715,5 @@ void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle) { } else { mgmtAddToShellQueue(ahandle); } -} \ No newline at end of file +} + diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index f0aa13ee3d..a6419d3549 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -20,7 +20,6 @@ #include "tlog.h" #include "tlosertree.h" #include "tscompression.h" -#include "tstatus.h" #include "ttime.h" #include "qast.h" diff --git a/src/util/inc/tstatus.h b/src/util/inc/tstatus.h deleted file mode 100644 index 4fa392ca35..0000000000 --- a/src/util/inc/tstatus.h +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TSTATUS_H -#define TDENGINE_TSTATUS_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include "taoserror.h" - - -enum _TSDB_DB_STATUS { - TSDB_DB_STATUS_READY, - TSDB_DB_STATUS_DROPPING, - TSDB_DB_STATUS_DROP_FROM_SDB -}; - -typedef enum _TSDB_VN_STATUS { - TSDB_VN_STATUS_NOT_READY, - TSDB_VN_STATUS_UNSYNCED, - TSDB_VN_STATUS_SLAVE, - TSDB_VN_STATUS_MASTER, - TSDB_VN_STATUS_CREATING, - TSDB_VN_STATUS_CLOSING, - TSDB_VN_STATUS_DELETING, -} EVnodeStatus; - -enum _TSDB_VN_SYNC_STATUS { - TSDB_VN_SYNC_STATUS_INIT, - TSDB_VN_SYNC_STATUS_SYNCING, - TSDB_VN_SYNC_STATUS_SYNC_CACHE, - TSDB_VN_SYNC_STATUS_SYNC_FILE -}; - -enum _TSDB_VN_DROP_STATUS { - TSDB_VN_DROP_STATUS_READY, - TSDB_VN_DROP_STATUS_DROPPING -}; - -enum _TSDB_MN_STATUS { - TSDB_MN_STATUS_OFFLINE, - TSDB_MN_STATUS_UNSYNCED, - TSDB_MN_STATUS_SYNCING, - TSDB_MN_STATUS_SERVING -}; - -enum _TSDB_MN_ROLE { - TSDB_MN_ROLE_UNDECIDED, - TSDB_MN_ROLE_SLAVE, - TSDB_MN_ROLE_MASTER -}; - -enum _TSDB_VG_STATUS { - TSDB_VG_STATUS_READY, - TSDB_VG_STATUS_UPDATE -}; - -enum _TSDB_VN_STREAM_STATUS { - TSDB_VN_STREAM_STATUS_STOP, - TSDB_VN_STREAM_STATUS_START -}; - -enum TSDB_TABLE_STATUS { - TSDB_METER_STATE_READY = 0x00, - TSDB_METER_STATE_INSERTING = 0x01, - TSDB_METER_STATE_IMPORTING = 0x02, - TSDB_METER_STATE_UPDATING = 0x04, - TSDB_METER_STATE_DROPPING = 0x10, - TSDB_METER_STATE_DROPPED = 0x18, -}; - -char* taosGetVgroupStatusStr(int32_t vgroupStatus); -char* taosGetDbStatusStr(int32_t dbStatus); -char* taosGetVnodeStatusStr(int32_t vnodeStatus); -char* taosGetVnodeSyncStatusStr(int32_t vnodeSyncStatus); -char* taosGetVnodeDropStatusStr(int32_t dropping); -char* taosGetDnodeLbStatusStr(int32_t dnodeBalanceStatus); -char* taosGetVgroupLbStatusStr(int32_t vglbStatus); -char* taosGetVnodeStreamStatusStr(int32_t vnodeStreamStatus); -char* taosGetTableStatusStr(int32_t tableStatus); -char *taosGetShowTypeStr(int32_t showType); -char *taosGetMnodeStatusStr(int32_t mnodeStatus); -char *taosGetMnodeRoleStr(int32_t mnodeRole); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TSTATUS_H diff --git a/src/util/src/tstatus.c b/src/util/src/tstatus.c deleted file mode 100644 index cfcbea0ca1..0000000000 --- a/src/util/src/tstatus.c +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "taosmsg.h" -#include "tstatus.h" - -char* taosGetVgroupStatusStr(int32_t vgroupStatus) { - switch (vgroupStatus) { - case TSDB_VG_STATUS_READY: return (char*)tstrerror(vgroupStatus); - case TSDB_VG_STATUS_IN_PROGRESS: return (char*)tstrerror(vgroupStatus); - case TSDB_VG_STATUS_NO_DISK_PERMISSIONS: return (char*)tstrerror(vgroupStatus); - case TSDB_VG_STATUS_SERVER_NO_PACE: return (char*)tstrerror(vgroupStatus); - case TSDB_VG_STATUS_SERV_OUT_OF_MEMORY: return (char*)tstrerror(vgroupStatus); - case TSDB_VG_STATUS_INIT_FAILED: return (char*)tstrerror(vgroupStatus); - case TSDB_VG_STATUS_FULL: return (char*)tstrerror(vgroupStatus); - default: return "undefined"; - } -} - -char* taosGetDbStatusStr(int32_t dbStatus) { - switch (dbStatus) { - case TSDB_DB_STATUS_READY: return "ready"; - case TSDB_DB_STATUS_DROPPING: return "dropping"; - case TSDB_DB_STATUS_DROP_FROM_SDB: return "drop_from_sdb"; - default: return "undefined"; - } -} - -char* taosGetVnodeStatusStr(int32_t vnodeStatus) { - switch (vnodeStatus) { - case TSDB_VN_STATUS_NOT_READY:return "not_ready"; - case TSDB_VN_STATUS_UNSYNCED: return "unsynced"; - case TSDB_VN_STATUS_SLAVE: return "slave"; - case TSDB_VN_STATUS_MASTER: return "master"; - case TSDB_VN_STATUS_CREATING: return "creating"; - case TSDB_VN_STATUS_CLOSING: return "closing"; - case TSDB_VN_STATUS_DELETING: return "deleting"; - default: return "undefined"; - } -} - -char* taosGetVnodeSyncStatusStr(int32_t vnodeSyncStatus) { - switch (vnodeSyncStatus) { - case TSDB_VN_SYNC_STATUS_INIT: return "ready"; - case TSDB_VN_SYNC_STATUS_SYNCING: return "syncing"; - case TSDB_VN_SYNC_STATUS_SYNC_CACHE: return "sync_cache"; - case TSDB_VN_SYNC_STATUS_SYNC_FILE: return "sync_file"; - default: return "undefined"; - } -} - -char* taosGetVnodeDropStatusStr(int32_t dropping) { - switch (dropping) { - case TSDB_VN_DROP_STATUS_READY: return "ready"; - case TSDB_VN_DROP_STATUS_DROPPING: return "dropping"; - default: return "undefined"; - } -} - -char* taosGetDnodeLbStatusStr(int32_t dnodeBalanceStatus) { - switch (dnodeBalanceStatus) { - case TSDB_DN_LB_STATUS_BALANCED: return "balanced"; - case TSDB_DN_LB_STATUS_BALANCING: return "balancing"; - case TSDB_DN_LB_STATUS_OFFLINE_REMOVING: return "offline removing"; - case TSDB_DN_LB_STATE_SHELL_REMOVING: return "removing"; - default: return "undefined"; - } -} - -char* taosGetVgroupLbStatusStr(int32_t vglbStatus) { - switch (vglbStatus) { - case TSDB_VG_LB_STATUS_READY: return "ready"; - case TSDB_VG_LB_STATUS_UPDATE: return "updating"; - default: return "undefined"; - } -} - -char* taosGetVnodeStreamStatusStr(int32_t vnodeStreamStatus) { - switch (vnodeStreamStatus) { - case TSDB_VN_STREAM_STATUS_START: return "start"; - case TSDB_VN_STREAM_STATUS_STOP: return "stop"; - default: return "undefined"; - } -} - -char* taosGetTableStatusStr(int32_t tableStatus) { - switch(tableStatus) { - case TSDB_METER_STATE_INSERTING:return "inserting"; - case TSDB_METER_STATE_IMPORTING:return "importing"; - case TSDB_METER_STATE_UPDATING: return "updating"; - case TSDB_METER_STATE_DROPPING: return "deleting"; - case TSDB_METER_STATE_DROPPED: return "dropped"; - case TSDB_METER_STATE_READY: return "ready"; - default:return "undefined"; - } -} - -char *taosGetShowTypeStr(int32_t showType) { - switch (showType) { - case TSDB_MGMT_TABLE_ACCT: return "show accounts"; - case TSDB_MGMT_TABLE_USER: return "show users"; - case TSDB_MGMT_TABLE_DB: return "show databases"; - case TSDB_MGMT_TABLE_TABLE: return "show tables"; - case TSDB_MGMT_TABLE_DNODE: return "show dnodes"; - case TSDB_MGMT_TABLE_MNODE: return "show mnodes"; - case TSDB_MGMT_TABLE_VGROUP: return "show vgroups"; - case TSDB_MGMT_TABLE_METRIC: return "show stables"; - case TSDB_MGMT_TABLE_MODULE: return "show modules"; - case TSDB_MGMT_TABLE_QUERIES: return "show queries"; - case TSDB_MGMT_TABLE_STREAMS: return "show streams"; - case TSDB_MGMT_TABLE_CONFIGS: return "show configs"; - case TSDB_MGMT_TABLE_CONNS: return "show connections"; - case TSDB_MGMT_TABLE_SCORES: return "show scores"; - case TSDB_MGMT_TABLE_GRANTS: return "show grants"; - case TSDB_MGMT_TABLE_VNODES: return "show vnodes"; - default: return "undefined"; - } -} - -char *taosGetMnodeStatusStr(int32_t mnodeStatus) { - switch (mnodeStatus) { - case TSDB_MN_STATUS_OFFLINE: return "offline"; - case TSDB_MN_STATUS_UNSYNCED: return "unsynced"; - case TSDB_MN_STATUS_SYNCING: return "syncing"; - case TSDB_MN_STATUS_SERVING: return "serving"; - default: return "undefined"; - } -} - -char *taosGetMnodeRoleStr(int32_t mnodeRole) { - switch (mnodeRole) { - case TSDB_MN_ROLE_UNDECIDED: return "undicided"; - case TSDB_MN_ROLE_SLAVE: return "slave"; - case TSDB_MN_ROLE_MASTER: return "master"; - default: return "undefined"; - } -} diff --git a/src/vnode/main/inc/vnodeInt.h b/src/vnode/main/inc/vnodeInt.h index 4d078869c4..96526d7209 100644 --- a/src/vnode/main/inc/vnodeInt.h +++ b/src/vnode/main/inc/vnodeInt.h @@ -23,18 +23,10 @@ extern "C" { #include "tsync.h" #include "twal.h" -typedef enum _VN_STATUS { - VN_STATUS_INIT, - VN_STATUS_CREATING, - VN_STATUS_READY, - VN_STATUS_CLOSING, - VN_STATUS_DELETING, -} EVnStatus; - typedef struct { int32_t vgId; // global vnode group ID int32_t refCount; // reference count - EVnStatus status; + int status; int role; int64_t version; void *wqueue; diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 1211828a47..786e6f63a0 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -20,7 +20,6 @@ #include "taosmsg.h" #include "tlog.h" #include "trpc.h" -#include "tstatus.h" #include "tsdb.h" #include "ttime.h" #include "ttimer.h" @@ -112,7 +111,7 @@ int32_t vnodeDrop(int32_t vgId) { } dTrace("pVnode:%p vgId:%d, vnode will be dropped", pVnode, pVnode->vgId); - pVnode->status = VN_STATUS_DELETING; + pVnode->status = TAOS_VN_STATUS_DELETING; vnodeCleanUp(pVnode); return TSDB_CODE_SUCCESS; @@ -124,7 +123,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { SVnodeObj *pVnode = calloc(sizeof(SVnodeObj), 1); pVnode->vgId = vnode; - pVnode->status = VN_STATUS_INIT; + pVnode->status = TAOS_VN_STATUS_INIT; pVnode->refCount = 1; pVnode->version = 0; taosAddIntHash(tsDnodeVnodesHash, pVnode->vgId, (char *)(&pVnode)); @@ -161,7 +160,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { walRestore(pVnode->wal, pVnode, vnodeWriteToQueue); - pVnode->status = VN_STATUS_READY; + pVnode->status = TAOS_VN_STATUS_READY; dTrace("pVnode:%p vgId:%d, vnode is opened in %s", pVnode, pVnode->vgId, rootDir); tsOpennedVnodes++; @@ -174,7 +173,7 @@ int32_t vnodeClose(int32_t vgId) { if (pVnode == NULL) return 0; dTrace("pVnode:%p vgId:%d, vnode will be closed", pVnode, pVnode->vgId); - pVnode->status = VN_STATUS_CLOSING; + pVnode->status = TAOS_VN_STATUS_CLOSING; vnodeCleanUp(pVnode); return 0; @@ -198,7 +197,7 @@ void vnodeRelease(void *pVnodeRaw) { dnodeFreeWqueue(pVnode->wqueue); pVnode->wqueue = NULL; - if (pVnode->status == VN_STATUS_DELETING) { + if (pVnode->status == TAOS_VN_STATUS_DELETING) { // remove the whole directory } @@ -258,7 +257,7 @@ void vnodeBuildStatusMsg(void *param) { static void vnodeBuildVloadMsg(char *pNode, void * param) { SVnodeObj *pVnode = *(SVnodeObj **) pNode; - if (pVnode->status == VN_STATUS_DELETING) return; + if (pVnode->status == TAOS_VN_STATUS_DELETING) return; SDMStatusMsg *pStatus = param; if (pStatus->openVnodes >= TSDB_MAX_VNODES) return; diff --git a/src/vnode/main/src/vnodeRead.c b/src/vnode/main/src/vnodeRead.c index 929a30fbcd..59f392eb82 100644 --- a/src/vnode/main/src/vnodeRead.c +++ b/src/vnode/main/src/vnodeRead.c @@ -42,7 +42,7 @@ int32_t vnodeProcessRead(void *param, int msgType, void *pCont, int32_t contLen, if (vnodeProcessReadMsgFp[msgType] == NULL) return TSDB_CODE_MSG_NOT_PROCESSED; - if (pVnode->status == VN_STATUS_DELETING || pVnode->status == VN_STATUS_CLOSING) + if (pVnode->status == TAOS_VN_STATUS_DELETING || pVnode->status == TAOS_VN_STATUS_CLOSING) return TSDB_CODE_NOT_ACTIVE_VNODE; return (*vnodeProcessReadMsgFp[msgType])(pVnode, pCont, contLen, ret); diff --git a/src/vnode/main/src/vnodeWrite.c b/src/vnode/main/src/vnodeWrite.c index c6699bd62c..605a57ebd0 100644 --- a/src/vnode/main/src/vnodeWrite.c +++ b/src/vnode/main/src/vnodeWrite.c @@ -41,18 +41,19 @@ void vnodeInitWriteFp(void) { vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = vnodeProcessDropStableMsg; } -int32_t vnodeProcessWrite(void *param, int qtype, SWalHead *pHead, void *item) { +int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { int32_t code = 0; - SVnodeObj *pVnode = (SVnodeObj *)param; + SVnodeObj *pVnode = (SVnodeObj *)param1; + SWalHead *pHead = param2; if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) return TSDB_CODE_MSG_NOT_PROCESSED; - if (pVnode->status == VN_STATUS_DELETING || pVnode->status == VN_STATUS_CLOSING) + if (pVnode->status == TAOS_VN_STATUS_DELETING || pVnode->status == TAOS_VN_STATUS_CLOSING) return TSDB_CODE_NOT_ACTIVE_VNODE; if (pHead->version == 0) { // from client - if (pVnode->status != VN_STATUS_READY) + if (pVnode->status != TAOS_VN_STATUS_READY) return TSDB_CODE_NOT_ACTIVE_VNODE; // if (pVnode->replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) From dc3c33a820f95657ff3138d51df54e84e6e31f2b Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 8 Apr 2020 21:05:19 +0800 Subject: [PATCH 07/78] integrate WAL module --- src/inc/dnode.h | 2 +- src/inc/tsync.h | 6 ++-- src/inc/twal.h | 4 +-- src/vnode/main/inc/vnodeInt.h | 4 +-- src/vnode/main/src/vnodeMain.c | 54 +++++++++++++++++++++++++++------ src/vnode/main/src/vnodeWrite.c | 3 +- src/vnode/wal/src/walMain.c | 12 ++++---- src/vnode/wal/test/waltest.c | 10 ++++-- 8 files changed, 69 insertions(+), 26 deletions(-) diff --git a/src/inc/dnode.h b/src/inc/dnode.h index fa5e3a3b3d..db39906c68 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -42,7 +42,7 @@ void *dnodeAllocateWqueue(void *pVnode); void dnodeFreeWqueue(void *queue); void *dnodeAllocateRqueue(void *pVnode); void dnodeFreeRqueue(void *rqueue); -void dnodeSendWriteResponse(void *pVnode, void *param, int32_t code); +void dnodeSendRpcWriteRsp(void *pVnode, void *param, int32_t code); #ifdef __cplusplus } diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 555e0503da..ea20ceec79 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -66,11 +66,11 @@ typedef struct { // if name is null, get the file from index or after, used by master // if name is provided, get the named file at the specified index, used by unsynced node // it returns the file magic number and size, if file not there, magic shall be 0. - uint32_t (*getFileInfo)(char *name, int *index, int *size); + uint32_t (*getFileInfo)(void *ahandle, char *name, uint32_t *index, int32_t *size); // get the wal file from index or after // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file - int (*getWalInfo)(char *name, int *index); + int (*getWalInfo)(void *ahandle, char *name, uint32_t *index); // when a forward pkt is received, call this to handle data int (*writeToCache)(void *ahandle, void *pHead, int type); @@ -94,11 +94,13 @@ int syncGetNodesRole(tsync_h shandle, SNodesRole *); extern char *syncRole[]; +//global configurable parameters extern int tsMaxSyncNum; extern int tsSyncTcpThreads; extern int tsMaxWatchFiles; extern short tsSyncPort; extern int tsMaxFwdInfo; +extern int sDebugFlag; #ifdef __cplusplus } diff --git a/src/inc/twal.h b/src/inc/twal.h index 3648f5ae29..53d4f835b0 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -40,12 +40,12 @@ typedef struct { typedef void* twal_h; // WAL HANDLE -twal_h walOpen(char *path, int max, int level); +twal_h walOpen(char *path, SWalCfg *pCfg); void walClose(twal_h); int walRenew(twal_h); int walWrite(twal_h, SWalHead *); void walFsync(twal_h); -int walRestore(twal_h, void *pVnode, int (*writeFp)(void *ahandle, SWalHead *pHead, int type)); +int walRestore(twal_h, void *pVnode, int (*writeFp)(void *ahandle, void *pHead, int type)); int walGetWalFile(twal_h, char *name, uint32_t *index); extern int wDebugFlag; diff --git a/src/vnode/main/inc/vnodeInt.h b/src/vnode/main/inc/vnodeInt.h index 4d078869c4..75d3117eac 100644 --- a/src/vnode/main/inc/vnodeInt.h +++ b/src/vnode/main/inc/vnodeInt.h @@ -35,7 +35,7 @@ typedef struct { int32_t vgId; // global vnode group ID int32_t refCount; // reference count EVnStatus status; - int role; + int8_t role; int64_t version; void *wqueue; void *rqueue; @@ -49,7 +49,7 @@ typedef struct { SWalCfg walCfg; } SVnodeObj; -int vnodeWriteToQueue(void *param, SWalHead *pHead, int type); +int vnodeWriteToQueue(void *param, void *pHead, int type); void vnodeInitWriteFp(void); void vnodeInitReadFp(void); diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 1211828a47..a3c60c4387 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -29,13 +29,18 @@ #include "vnode.h" #include "vnodeInt.h" -static void *tsDnodeVnodesHash; -static void vnodeCleanUp(SVnodeObj *pVnode); -static void vnodeBuildVloadMsg(char *pNode, void * param); -static int vnodeWALCallback(void *arg); -static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg); -static int32_t vnodeReadCfg(SVnodeObj *pVnode); +static void *tsDnodeVnodesHash; +static void vnodeCleanUp(SVnodeObj *pVnode); +static void vnodeBuildVloadMsg(char *pNode, void * param); +static int vnodeWalCallback(void *arg); +static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg); +static int32_t vnodeReadCfg(SVnodeObj *pVnode); +static int vnodeWalCallback(void *arg); +static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size); +static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index); +static void vnodeNotifyRole(void *ahandle, int8_t role); +// module global static int tsOpennedVnodes; static pthread_once_t vnodeModuleInit = PTHREAD_ONCE_INIT; @@ -140,14 +145,27 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->rqueue = dnodeAllocateRqueue(pVnode); sprintf(temp, "%s/wal", rootDir); - pVnode->wal = walOpen(temp, pVnode->walCfg.wals, pVnode->walCfg.commitLog); - pVnode->sync = NULL; + pVnode->wal = walOpen(temp, &pVnode->walCfg); + + SSyncInfo syncInfo; + syncInfo.vgId = pVnode->vgId; + syncInfo.vgId = pVnode->version; + syncInfo.syncCfg = pVnode->syncCfg; + sprintf(syncInfo.path, "%s/tsdb/", rootDir); + syncInfo.ahandle = pVnode; + syncInfo.getWalInfo = vnodeGetWalInfo; + syncInfo.getFileInfo = vnodeGetFileInfo; + syncInfo.writeToCache = vnodeWriteToQueue; + syncInfo.confirmForward = dnodeSendRpcWriteRsp; + syncInfo.notifyRole = vnodeNotifyRole; + // pVnode->sync = syncStart(&syncInfo);; + pVnode->events = NULL; pVnode->cq = NULL; STsdbAppH appH = {0}; appH.appH = (void *)pVnode; - appH.walCallBack = vnodeWALCallback; + appH.walCallBack = vnodeWalCallback; sprintf(temp, "%s/tsdb", rootDir); void *pTsdb = tsdbOpenRepo(temp, &appH); @@ -281,11 +299,27 @@ static void vnodeCleanUp(SVnodeObj *pVnode) { } // TODO: this is a simple implement -static int vnodeWALCallback(void *arg) { +static int vnodeWalCallback(void *arg) { SVnodeObj *pVnode = arg; return walRenew(pVnode->wal); } +static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size) { + // SVnodeObj *pVnode = ahandle; + //tsdbGetFileInfo(pVnode->tsdb, name, index, size); + return 0; +} + +static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index) { + SVnodeObj *pVnode = ahandle; + return walGetWalFile(pVnode->wal, name, index); +} + +static void vnodeNotifyRole(void *ahandle, int8_t role) { + SVnodeObj *pVnode = ahandle; + pVnode->role = role; +} + static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { char cfgFile[TSDB_FILENAME_LEN * 2] = {0}; sprintf(cfgFile, "%s/vnode%d/config", tsVnodeDir, pVnodeCfg->cfg.vgId); diff --git a/src/vnode/main/src/vnodeWrite.c b/src/vnode/main/src/vnodeWrite.c index c6699bd62c..fc73850f40 100644 --- a/src/vnode/main/src/vnodeWrite.c +++ b/src/vnode/main/src/vnodeWrite.c @@ -248,8 +248,9 @@ static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet return code; } -int vnodeWriteToQueue(void *param, SWalHead *pHead, int type) { +int vnodeWriteToQueue(void *param, void *data, int type) { SVnodeObj *pVnode = param; + SWalHead *pHead = data; int size = sizeof(SWalHead) + pHead->len; SWalHead *pWal = (SWalHead *)taosAllocateQitem(size); diff --git a/src/vnode/wal/src/walMain.c b/src/vnode/wal/src/walMain.c index 99de5cceda..7c5602680f 100644 --- a/src/vnode/wal/src/walMain.c +++ b/src/vnode/wal/src/walMain.c @@ -49,18 +49,18 @@ int wDebugFlag = 135; static uint32_t walSignature = 0xFAFBFDFE; static int walHandleExistingFiles(char *path); -static int walRestoreWalFile(char *name, void *pVnode, int (*writeFp)(void *, SWalHead *, int)); +static int walRestoreWalFile(char *name, void *pVnode, int (*writeFp)(void *, void *, int)); static int walRemoveWalFiles(char *path); -void *walOpen(char *path, int max, int level) { +void *walOpen(char *path, SWalCfg *pCfg) { SWal *pWal = calloc(sizeof(SWal), 1); if (pWal == NULL) return NULL; pWal->fd = -1; - pWal->max = max; + pWal->max = pCfg->wals; pWal->id = 0; pWal->num = 0; - pWal->level = level; + pWal->level = pCfg->commitLog; strcpy(pWal->path, path); pthread_mutex_init(&pWal->mutex, NULL); @@ -169,7 +169,7 @@ void walFsync(void *handle) { fsync(pWal->fd); } -int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, SWalHead *, int)) { +int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int)) { SWal *pWal = (SWal *)handle; int code = 0; struct dirent *ent; @@ -246,7 +246,7 @@ int walGetWalFile(void *handle, char *name, uint32_t *index) { return code; } -static int walRestoreWalFile(char *name, void *pVnode, int (*writeFp)(void *, SWalHead *, int)) { +static int walRestoreWalFile(char *name, void *pVnode, int (*writeFp)(void *, void *, int)) { int code = 0; char *buffer = malloc(1024000); // size for one record diff --git a/src/vnode/wal/test/waltest.c b/src/vnode/wal/test/waltest.c index 37d1d8e84c..8e10bc11e5 100644 --- a/src/vnode/wal/test/waltest.c +++ b/src/vnode/wal/test/waltest.c @@ -21,8 +21,10 @@ int64_t ver = 0; void *pWal = NULL; -int writeToQueue(void *pVnode, SWalHead *pHead, int type) { +int writeToQueue(void *pVnode, void *data, int type) { // do nothing + SWalHead *pHead = data; + if (pHead->version > ver) ver = pHead->version; @@ -74,7 +76,11 @@ int main(int argc, char *argv[]) { taosInitLog("wal.log", 100000, 10); - pWal = walOpen(path, max, level); + SWalCfg walCfg; + walCfg.commitLog = level; + walCfg.wals = max; + + pWal = walOpen(path, &walCfg); if (pWal == NULL) { printf("failed to open wal\n"); exit(-1); From 0ba919dbacadd0a0d0890c34461e24478326b209 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 8 Apr 2020 21:16:58 +0800 Subject: [PATCH 08/78] add compile option for sync --- src/dnode/CMakeLists.txt | 2 +- src/vnode/main/src/vnodeMain.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index 4bd89e238e..a81f8c0c9d 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -28,7 +28,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) ENDIF () IF (TD_VPEER) - TARGET_LINK_LIBRARIES(taosd balance) + TARGET_LINK_LIBRARIES(taosd balance sync) ENDIF () SET(PREPARE_ENV_CMD "prepare_env_cmd") diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index a3c60c4387..d51b9ff846 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -158,7 +158,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { syncInfo.writeToCache = vnodeWriteToQueue; syncInfo.confirmForward = dnodeSendRpcWriteRsp; syncInfo.notifyRole = vnodeNotifyRole; - // pVnode->sync = syncStart(&syncInfo);; + pVnode->sync = syncStart(&syncInfo);; pVnode->events = NULL; pVnode->cq = NULL; From 769ef5be3c22ad80e9513f1960eeedf5461b51de Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 8 Apr 2020 22:19:46 +0800 Subject: [PATCH 09/78] remove the hard code for max number of vnodes --- src/dnode/src/dnodeMgmt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 5f1e7a7a94..4f9f03f179 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -132,7 +132,7 @@ static int32_t dnodeOpenVnodes() { char vnodeDir[TSDB_FILENAME_LEN * 3]; int32_t failed = 0; - int32_t *vnodeList = (int32_t *)malloc(sizeof(int32_t) * 10000); + int32_t *vnodeList = (int32_t *)malloc(sizeof(int32_t) * TSDB_MAX_VNODES); int32_t numOfVnodes = dnodeGetVnodeList(vnodeList); for (int32_t i = 0; i < numOfVnodes; ++i) { @@ -147,7 +147,7 @@ static int32_t dnodeOpenVnodes() { } static void dnodeCloseVnodes() { - int32_t *vnodeList = (int32_t *)malloc(sizeof(int32_t) * 10000); + int32_t *vnodeList = (int32_t *)malloc(sizeof(int32_t) * TSDB_MAX_VNODES); int32_t numOfVnodes = dnodeGetVnodeList(vnodeList); for (int32_t i = 0; i < numOfVnodes; ++i) { From 366068e287ca12f7ddc5292b8045c54d0a17128a Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 8 Apr 2020 22:34:05 +0800 Subject: [PATCH 10/78] make the add/sub atomic --- src/vnode/main/src/vnodeMain.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index d51b9ff846..3211c0aff5 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -41,7 +41,7 @@ static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index); static void vnodeNotifyRole(void *ahandle, int8_t role); // module global -static int tsOpennedVnodes; +static int32_t tsOpennedVnodes; static pthread_once_t vnodeModuleInit = PTHREAD_ONCE_INIT; static void vnodeInit() { @@ -182,7 +182,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->status = VN_STATUS_READY; dTrace("pVnode:%p vgId:%d, vnode is opened in %s", pVnode, pVnode->vgId, rootDir); - tsOpennedVnodes++; + atomic_add_fetch_32(&tsOpennedVnodes, 1); return TSDB_CODE_SUCCESS; } @@ -223,8 +223,8 @@ void vnodeRelease(void *pVnodeRaw) { dTrace("pVnode:%p vgId:%d, vnode is released", pVnode, pVnode->vgId); free(pVnode); - tsOpennedVnodes--; - if (tsOpennedVnodes <= 0) { + int32_t count = atomic_sub_fetch_32(&tsOpennedVnodes, 1); + if (count <= 0) { taosCleanUpIntHash(tsDnodeVnodesHash); vnodeModuleInit = PTHREAD_ONCE_INIT; tsDnodeVnodesHash = NULL; From 4926dddebb203dc7016bece9e59ac2c33ebd95af Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 8 Apr 2020 23:14:44 +0800 Subject: [PATCH 11/78] fix kill script to avoid others' path with taosdata be killed by mistake. --- tests/script/sh/stop_dnodes.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh index acdfbe9e6a..fe4205824a 100755 --- a/tests/script/sh/stop_dnodes.sh +++ b/tests/script/sh/stop_dnodes.sh @@ -6,8 +6,8 @@ if [ -n "$PID" ]; then sudo systemctl stop taosd fi -PID=`ps -ef|grep taosd | grep -v grep | awk '{print $2}'` +PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` if [ -n "$PID" ]; then echo sudo kill -9 $PID - sudo kill -9 $PID + sudo pkill taosd fi From 4f350284abafcd954ea36b04815c3c2cb539ac01 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 8 Apr 2020 23:36:37 +0800 Subject: [PATCH 12/78] [TD-82] first edition of balance --- src/dnode/src/dnodeMain.c | 1 - src/inc/tcluster.h | 10 +++++----- src/mnode/src/mgmtDnode.c | 14 +++++++------- src/mnode/src/mgmtSdb.c | 4 ++-- src/mnode/src/mgmtVgroup.c | 2 +- src/util/inc/tmodule.h | 2 -- src/vnode/main/src/vnodeMain.c | 1 + 7 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index d95a344058..01e2c4dfcc 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -220,7 +220,6 @@ static int32_t dnodeInitStorage() { sprintf(tsMnodeDir, "%s/mnode", dataDir); sprintf(tsVnodeDir, "%s/vnode", dataDir); sprintf(tsDnodeDir, "%s/dnode", dataDir); - mkdir(tsMnodeDir, 0755); mkdir(tsVnodeDir, 0755); mkdir(tsDnodeDir, 0755); diff --git a/src/inc/tcluster.h b/src/inc/tcluster.h index 60436ee743..28467ca683 100644 --- a/src/inc/tcluster.h +++ b/src/inc/tcluster.h @@ -26,11 +26,11 @@ extern "C" { struct _dnode_obj; -enum _TSDB_DN_STATUS { - TSDB_DN_STATUS_OFFLINE, - TSDB_DN_STATUS_DROPING, - TSDB_DN_STATUS_BALANCING, - TSDB_DN_STATUS_READY +enum _TAOS_DN_STATUS { + TAOS_DN_STATUS_OFFLINE, + TAOS_DN_STATUS_DROPPING, + TAOS_DN_STATUS_BALANCING, + TAOS_DN_STATUS_READY }; int32_t clusterInit(); diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index b60d1cb0ba..e6d186ac1b 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -48,7 +48,7 @@ int32_t clusterInitDnodes() { tsDnodeObj.publicIp = inet_addr(tsPublicIp); tsDnodeObj.createdTime = taosGetTimestampMs(); tsDnodeObj.numOfTotalVnodes = tsNumOfTotalVnodes; - tsDnodeObj.status = TSDB_DN_STATUS_OFFLINE; + tsDnodeObj.status = TAOS_DN_STATUS_OFFLINE; tsDnodeObj.lastReboot = taosGetTimestampSec(); sprintf(tsDnodeObj.dnodeName, "%d", tsDnodeObj.dnodeId); @@ -203,9 +203,9 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { mgmtReleaseVgroup(pVgroup); } - if (pDnode->status != TSDB_DN_STATUS_READY) { + if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { mTrace("dnode:%d, from offline to online", pDnode->dnodeId); - pDnode->status = TSDB_DN_STATUS_READY; + pDnode->status = TAOS_DN_STATUS_READY; balanceNotify(); } @@ -673,10 +673,10 @@ static int32_t clusterRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, char* clusterGetDnodeStatusStr(int32_t dnodeStatus) { switch (dnodeStatus) { - case TSDB_DN_STATUS_OFFLINE: return "offline"; - case TSDB_DN_STATUS_DROPING: return "dropping"; - case TSDB_DN_STATUS_BALANCING: return "balancing"; - case TSDB_DN_STATUS_READY: return "ready"; + case TAOS_DN_STATUS_OFFLINE: return "offline"; + case TAOS_DN_STATUS_DROPPING: return "dropping"; + case TAOS_DN_STATUS_BALANCING: return "balancing"; + case TAOS_DN_STATUS_READY: return "ready"; default: return "undefined"; } } diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 06cb9dca5c..60614157f1 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -435,7 +435,7 @@ void sdbIncRef(void *handle, void *pRow) { SSdbTable *pTable = handle; int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos); atomic_add_fetch_32(pRefCount, 1); - if (0) { + if (0 && strcmp(pTable->tableName, "dnodes") == 0) { sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); } } @@ -446,7 +446,7 @@ void sdbDecRef(void *handle, void *pRow) { SSdbTable *pTable = handle; int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos); int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); - if (0) { + if (0 && strcmp(pTable->tableName, "dnodes") == 0) { sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); } int8_t* updateEnd = pRow + pTable->refCountPos - 1; diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index a0b80c8ea3..4c969124a0 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -372,7 +372,7 @@ char *mgmtGetVnodeStatus(SVgObj *pVgroup, SVnodeGid *pVnode) { } clusterReleaseDnode(pDnode); - if (pDnode->status == TSDB_DN_STATUS_OFFLINE) { + if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { return "offline"; } diff --git a/src/util/inc/tmodule.h b/src/util/inc/tmodule.h index 9f8ef147f8..f5ad6a6b60 100644 --- a/src/util/inc/tmodule.h +++ b/src/util/inc/tmodule.h @@ -28,8 +28,6 @@ enum _module { TSDB_MOD_MGMT, TSDB_MOD_HTTP, TSDB_MOD_MONITOR, - TSDB_MOD_DCLUSTER, - TSDB_MOD_MSTORAGE, TSDB_MOD_MAX }; diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 786e6f63a0..4c2d86a518 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -266,6 +266,7 @@ static void vnodeBuildVloadMsg(char *pNode, void * param) { pLoad->vgId = htonl(pVnode->vgId); pLoad->vnode = htonl(pVnode->vgId); pLoad->status = pVnode->status; + pLoad->role = pVnode->role; } static void vnodeCleanUp(SVnodeObj *pVnode) { From b37185e5403c514d5b80208bc92ae53408c6ccc8 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 9 Apr 2020 04:30:33 +0000 Subject: [PATCH 13/78] fix part of mem-leak --- src/common/src/dataformat.c | 2 +- src/vnode/main/src/vnodeWrite.c | 3 +++ src/vnode/tsdb/src/tsdbFile.c | 1 + src/vnode/tsdb/src/tsdbMain.c | 8 ++++++-- src/vnode/tsdb/src/tsdbMeta.c | 13 ++++++++++++- src/vnode/tsdb/src/tsdbMetaFile.c | 1 + 6 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index bff041df1b..45850d1788 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -141,7 +141,7 @@ STSchema *tdDupSchema(STSchema *pSchema) { * Free the SSchema object created by tdNewSchema or tdDupSchema */ void tdFreeSchema(STSchema *pSchema) { - if (pSchema == NULL) free(pSchema); + if (pSchema != NULL) free(pSchema); } /** diff --git a/src/vnode/main/src/vnodeWrite.c b/src/vnode/main/src/vnodeWrite.c index c6699bd62c..648e7197b0 100644 --- a/src/vnode/main/src/vnodeWrite.c +++ b/src/vnode/main/src/vnodeWrite.c @@ -25,6 +25,7 @@ #include "dataformat.h" #include "vnode.h" #include "vnodeInt.h" +#include "tutil.h" static int32_t (*vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *, void *, SRspRet *); static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pMsg, SRspRet *); @@ -157,6 +158,8 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe void *pTsdb = vnodeGetTsdb(pVnode); code = tsdbCreateTable(pTsdb, &tCfg); + tfree(pDestSchema); + dTrace("pVnode:%p vgId:%d, table:%s is created, result:%x", pVnode, pVnode->vgId, pTable->tableId, code); return code; } diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index bd6699eb84..8bdfe63002 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -62,6 +62,7 @@ STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles) { // TODO } } + closedir(dir); return pFileH; } diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 61f4995e43..aa185f4bf2 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -268,6 +268,9 @@ int32_t tsdbCloseRepo(tsdb_repo_t *repo) { tsdbFreeCache(pRepo->tsdbCache); + tfree(pRepo->rootDir); + tfree(pRepo); + return 0; } @@ -847,6 +850,7 @@ static void *tsdbCommitData(void *arg) { tsdbLockRepo(arg); tdListMove(pCache->imem->list, pCache->pool.memPool); + tdListFree(pCache->imem->list); free(pCache->imem); pCache->imem = NULL; pRepo->commit = 0; @@ -1125,11 +1129,11 @@ static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsTo *len += pCompCol->len; } - if (pCompData == NULL) free((void *)pCompData); + tfree(pCompData); return 0; _err: - if (pCompData == NULL) free((void *)pCompData); + tfree(pCompData); return -1; } diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index 0c6fc61701..22f7357135 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -312,6 +312,14 @@ int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId) { // return 0; // } +static void tsdbFreeMemTable(SMemTable *pMemTable) { + if (pMemTable) { + tSkipListDestroy(pMemTable->pData); + } + + free(pMemTable); +} + static int tsdbFreeTable(STable *pTable) { // TODO: finish this function if (pTable->type == TSDB_CHILD_TABLE) { @@ -323,7 +331,10 @@ static int tsdbFreeTable(STable *pTable) { // Free content if (TSDB_TABLE_IS_SUPER_TABLE(pTable)) { tSkipListDestroy(pTable->pIndex); - } + } + + tsdbFreeMemTable(pTable->mem); + tsdbFreeMemTable(pTable->imem); free(pTable); return 0; diff --git a/src/vnode/tsdb/src/tsdbMetaFile.c b/src/vnode/tsdb/src/tsdbMetaFile.c index d3cff1772c..6821fc2d98 100644 --- a/src/vnode/tsdb/src/tsdbMetaFile.c +++ b/src/vnode/tsdb/src/tsdbMetaFile.c @@ -177,6 +177,7 @@ void tsdbCloseMetaFile(SMetaFile *mfh) { close(mfh->fd); taosHashCleanup(mfh->map); + tfree(mfh); } static int32_t tsdbGetMetaFileName(char *rootDir, char *fname) { From 64e59d94630e4a56464cbbdead2eff52041b12a9 Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 9 Apr 2020 14:38:00 +0800 Subject: [PATCH 14/78] fix log print error --- src/util/src/tstring.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/util/src/tstring.c b/src/util/src/tstring.c index 7aca939f47..5a134df129 100644 --- a/src/util/src/tstring.c +++ b/src/util/src/tstring.c @@ -110,6 +110,8 @@ char *taosMsg[] = { "", "", "", + "", + "", "", //90 "config-table", From 3899ae87075e4542357313df60def450ebfab045 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Thu, 9 Apr 2020 15:57:32 +0800 Subject: [PATCH 15/78] fix the bug in sync integration --- src/rpc/src/rpcMain.c | 5 ++++- src/vnode/main/src/vnodeMain.c | 4 ++-- src/vnode/main/src/vnodeWrite.c | 5 +---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 6f86c2dd7c..a241a8ce07 100755 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -934,6 +934,9 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { memcpy(&pContext->ipSet, pHead->content, sizeof(pContext->ipSet)); tTrace("%s %p, redirect is received, numOfIps:%d", pRpc->label, pConn, pContext->ipSet.numOfIps); rpcSendReqToServer(pRpc, pContext); + } else if (pHead->code == TSDB_CODE_NOT_READY) { + pConn->pContext->code = pHead->code; + rpcProcessConnError(pConn->pContext, NULL); } else { rpcNotifyClient(pContext, &rpcMsg); } @@ -1078,7 +1081,7 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) { if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16)) tTrace( "%s %p, %s is sent to %s:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerIpstr, pConn->peerPort, - pHead->code, msgLen, pHead->sourceId, pHead->destId, pHead->tranId); + htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } writtenLen = (*taosSendData[pConn->connType])(pConn->peerIp, pConn->peerPort, pHead, msgLen, pConn->chandle); diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index bbc17c3440..f5a780214b 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -148,7 +148,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { SSyncInfo syncInfo; syncInfo.vgId = pVnode->vgId; - syncInfo.vgId = pVnode->version; + syncInfo.version = pVnode->version; syncInfo.syncCfg = pVnode->syncCfg; sprintf(syncInfo.path, "%s/tsdb/", rootDir); syncInfo.ahandle = pVnode; @@ -370,7 +370,7 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT; if (strcmp(option[0], "arbitratorIp") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; if (arbitratorIp == -1) return TSDB_CODE_INVALID_FILE_FORMAT; - pVnode->syncCfg.arbitratorIp = arbitratorIp; + pVnode->syncCfg.arbitratorIp = 0; int32_t quorum = -1; num = fscanf(fp, "%s %d", option[0], &quorum); diff --git a/src/vnode/main/src/vnodeWrite.c b/src/vnode/main/src/vnodeWrite.c index 849f5f95da..1504c92151 100644 --- a/src/vnode/main/src/vnodeWrite.c +++ b/src/vnode/main/src/vnodeWrite.c @@ -77,11 +77,8 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, item); if (code < 0) return code; -/* forward - if (pVnode->replica > 1 && pVnode->role == TAOS_SYNC_ROLE_MASTER) { + if (pVnode->syncCfg.replica > 1) code = syncForwardToPeer(pVnode->sync, pHead, item); - } -*/ return code; } From d5de8dc1e3ca4843994f0417d674fba34c3048a8 Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 9 Apr 2020 16:15:21 +0800 Subject: [PATCH 16/78] [TD-82] fix error while balance --- src/inc/tcluster.h | 1 + src/mnode/src/mgmtDnode.c | 6 +++--- src/mnode/src/mgmtSdb.c | 5 +++++ src/util/src/tglobalcfg.c | 7 +++++++ 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/inc/tcluster.h b/src/inc/tcluster.h index 28467ca683..769a819b90 100644 --- a/src/inc/tcluster.h +++ b/src/inc/tcluster.h @@ -36,6 +36,7 @@ enum _TAOS_DN_STATUS { int32_t clusterInit(); void clusterCleanUp(); char* clusterGetDnodeStatusStr(int32_t dnodeStatus); +bool clusterCheckModuleInDnode(struct _dnode_obj *pDnode, int moduleType); int32_t clusterInitDnodes(); void clusterCleanupDnodes(); diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index e6d186ac1b..e4ab114090 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -323,8 +323,7 @@ static int32_t clusterRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, char ipstr[32]; while (numOfRows < rows) { - clusterReleaseDnode(pDnode); - pShow->pNode = clusterGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); + pShow->pNode = clusterGetNextDnode(pShow->pNode, &pDnode); if (pDnode == NULL) break; cols = 0; @@ -366,13 +365,14 @@ static int32_t clusterRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, #endif numOfRows++; + clusterReleaseDnode(pDnode); } pShow->numOfReads += numOfRows; return numOfRows; } -static bool clusterCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) { +bool clusterCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) { uint32_t status = pDnode->moduleStatus & (1 << moduleType); return status > 0; } diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 60614157f1..ef2a23cd12 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -520,6 +520,11 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { if (pTable->keyType == SDB_KEY_TYPE_AUTO) { *((uint32_t *)pOper->pObj) = ++pTable->autoIndex; + + // let vgId increase from 2 + if (pTable->autoIndex == 1 && strcmp(pTable->tableName, "vgroups") == 0) { + *((uint32_t *)pOper->pObj) = ++pTable->autoIndex; + } } pTable->version++; sdbVersion++; diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index 8a0d66068e..88a3be0e02 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -111,7 +111,12 @@ short tsDaysPerFile = 10; int tsDaysToKeep = 3650; int tsReplications = TSDB_REPLICA_MIN_NUM; +#ifdef _MPEER int tsNumOfMPeers = 3; +#else +int tsNumOfMPeers = 1; +#endif + int tsMaxShellConns = 2000; int tsMaxTables = 100000; @@ -552,9 +557,11 @@ static void doInitGlobalConfig() { tsInitConfigOption(cfg++, "tblocks", &tsNumOfBlocksPerMeter, TSDB_CFG_VTYPE_SHORT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 32, 4096, 0, TSDB_CFG_UTYPE_NONE); +#ifdef _MPEER tsInitConfigOption(cfg++, "numOfMPeers", &tsNumOfMPeers, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, 1, 3, 0, TSDB_CFG_UTYPE_NONE); +#endif tsInitConfigOption(cfg++, "balanceInterval", &tsBalanceStartInterval, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, 1, 30000, 0, TSDB_CFG_UTYPE_NONE); From 7b5c13a857950b81233c32ea86968d685822bfba Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 9 Apr 2020 17:08:11 +0800 Subject: [PATCH 17/78] add query in shared library [TD-96] --- src/client/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 55fa45475a..ecbef79ae4 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -22,7 +22,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) # generate dynamic library (*.so) ADD_LIBRARY(taos SHARED ${SRC}) - TARGET_LINK_LIBRARIES(taos common trpc tutil pthread m rt) + TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m rt) SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) #set version of .so From 55fa8b060ab1350a35fa0b06a522ffa050f8591b Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 9 Apr 2020 17:49:35 +0800 Subject: [PATCH 18/78] fix compile while intergreate sync module --- src/client/src/tscServer.c | 3 ++- src/vnode/main/src/vnodeMain.c | 26 ++++++++++++++++---------- tests/script/general/table/basic2.sim | 2 +- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 3042e3e59b..46c7dd589b 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -517,7 +517,8 @@ int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pRetrieveMsg->free = htons(pQueryInfo->type); pMsg += sizeof(pQueryInfo->type); - pRetrieveMsg->header.vgId = htonl(1); + STableMeta* pTableMeta = pQueryInfo->pTableMetaInfo[0]->pTableMeta; + pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId); pMsg += sizeof(SRetrieveTableMsg); pRetrieveMsg->header.contLen = htonl(pSql->cmd.payloadLen); diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 987fd69a70..182b4b6257 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -28,6 +28,7 @@ #include "vnode.h" #include "vnodeInt.h" +static int32_t tsOpennedVnodes; static void *tsDnodeVnodesHash; static void vnodeCleanUp(SVnodeObj *pVnode); static void vnodeBuildVloadMsg(char *pNode, void * param); @@ -41,6 +42,11 @@ static void vnodeNotifyRole(void *ahandle, int8_t role); static pthread_once_t vnodeModuleInit = PTHREAD_ONCE_INIT; +#ifndef _VPEER +tsync_h syncStart(SSyncInfo *info) { return NULL; } +int syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle) { return 0; } +#endif + static void vnodeInit() { vnodeInitWriteFp(); vnodeInitReadFp(); @@ -56,7 +62,6 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { pthread_once(&vnodeModuleInit, vnodeInit); SVnodeObj *pTemp = (SVnodeObj *)taosGetIntHashData(tsDnodeVnodesHash, pVnodeCfg->cfg.vgId); - if (pTemp != NULL) { dPrint("vgId:%d, vnode already exist, pVnode:%p", pVnodeCfg->cfg.vgId, pTemp); return TSDB_CODE_SUCCESS; @@ -106,12 +111,13 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { } int32_t vnodeDrop(int32_t vgId) { - SVnodeObj *pVnode = *(SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId); - if (pVnode == NULL) { + SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId); + if (ppVnode == NULL || *ppVnode == NULL) { dTrace("vgId:%d, failed to drop, vgId not exist", vgId); return TSDB_CODE_INVALID_VGROUP_ID; } + SVnodeObj *pVnode = *ppVnode; dTrace("pVnode:%p vgId:%d, vnode will be dropped", pVnode, pVnode->vgId); pVnode->status = TAOS_VN_STATUS_DELETING; vnodeCleanUp(pVnode); @@ -183,10 +189,10 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { } int32_t vnodeClose(int32_t vgId) { + SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId); + if (ppVnode == NULL || *ppVnode == NULL) return 0; - SVnodeObj *pVnode = *(SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId); - if (pVnode == NULL) return 0; - + SVnodeObj *pVnode = *ppVnode; dTrace("pVnode:%p vgId:%d, vnode will be closed", pVnode, pVnode->vgId); pVnode->status = TAOS_VN_STATUS_CLOSING; vnodeCleanUp(pVnode); @@ -228,13 +234,13 @@ void vnodeRelease(void *pVnodeRaw) { } void *vnodeGetVnode(int32_t vgId) { - SVnodeObj *pVnode = *(SVnodeObj **) taosGetIntHashData(tsDnodeVnodesHash, vgId); - if (pVnode == NULL) { + SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId); + if (ppVnode == NULL || *ppVnode == NULL) { terrno = TSDB_CODE_INVALID_VGROUP_ID; - return NULL; + assert(false); } - return pVnode; + return *ppVnode; } void *vnodeAccquireVnode(int32_t vgId) { diff --git a/tests/script/general/table/basic2.sim b/tests/script/general/table/basic2.sim index 7701ca1c1f..73b17471e5 100644 --- a/tests/script/general/table/basic2.sim +++ b/tests/script/general/table/basic2.sim @@ -29,7 +29,7 @@ if $data03 != 1 then endi sql show d1.vgroups -if $data00 != 1 then +if $data00 != 2 then return -1 endi From 0ba8636c848c2ce1c5f2c23bcd1767b94e6184fb Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 9 Apr 2020 17:52:52 +0800 Subject: [PATCH 19/78] add some scripts --- tests/script/general/db/basic4.sim | 4 ++-- tests/script/general/db/basic5.sim | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/script/general/db/basic4.sim b/tests/script/general/db/basic4.sim index f620112cfb..deac9d47b4 100644 --- a/tests/script/general/db/basic4.sim +++ b/tests/script/general/db/basic4.sim @@ -36,7 +36,7 @@ sql show d1.vgroups if $rows != 1 then return -1 endi -if $data00 != 1 then +if $data00 != 2 then return -1 endi if $data01 != 4 then @@ -75,7 +75,7 @@ sql show d1.vgroups if $rows != 1 then return -1 endi -if $data00 != 1 then +if $data00 != 2 then return -1 endi if $data01 != 3 then diff --git a/tests/script/general/db/basic5.sim b/tests/script/general/db/basic5.sim index ec619389bd..ebb7d3c967 100644 --- a/tests/script/general/db/basic5.sim +++ b/tests/script/general/db/basic5.sim @@ -36,7 +36,7 @@ sql show d1.vgroups if $rows != 1 then return -1 endi -if $data00 != 1 then +if $data00 != 2 then return -1 endi if $data01 != 4 then From 37b293720337b7cf443b4cec4e889b0dcabf9d64 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 9 Apr 2020 19:08:26 +0800 Subject: [PATCH 20/78] fix stop_dnodes.sh with postpone socket cleanup and same timestamp issue in general/table/basic2.sim. --- tests/script/basicSuite.sim | 4 ++-- tests/script/general/table/basic1.sim | 1 - tests/script/general/table/basic2.sim | 5 +++-- tests/script/sh/stop_dnodes.sh | 8 +++++--- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index d5892d8682..290472b15d 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -2,7 +2,7 @@ run general/table/basic1.sim -#run general/table/basic2.sim -#run general/table/basic3.sim +run general/table/basic2.sim +run general/table/basic3.sim ################################## diff --git a/tests/script/general/table/basic1.sim b/tests/script/general/table/basic1.sim index fb924ac453..a8dfb0f21a 100644 --- a/tests/script/general/table/basic1.sim +++ b/tests/script/general/table/basic1.sim @@ -1,6 +1,5 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system ifconfig system sh/exec.sh -n dnode1 -s start sql connect diff --git a/tests/script/general/table/basic2.sim b/tests/script/general/table/basic2.sim index 7701ca1c1f..2afd6baf36 100644 --- a/tests/script/general/table/basic2.sim +++ b/tests/script/general/table/basic2.sim @@ -43,11 +43,12 @@ sql_error insert into d1.n2 values(now, 1) print =============== insert data2 sql insert into d1.n3 values(now, 1) -sql insert into d1.n3 values(now, 2) -sql insert into d1.n3 values(now, 3) +sql insert into d1.n3 values(now+1s, 2) +sql insert into d1.n3 values(now+2s, 3) print =============== query data sql select * from d1.n3 +print $rows if $rows != 3 then return -1 endi diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh index fe4205824a..a873eb5654 100755 --- a/tests/script/sh/stop_dnodes.sh +++ b/tests/script/sh/stop_dnodes.sh @@ -7,7 +7,9 @@ if [ -n "$PID" ]; then fi PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` -if [ -n "$PID" ]; then +while [ -n "$PID" ]; do echo sudo kill -9 $PID - sudo pkill taosd -fi + sudo pkill -9 taosd + sudo fuser -k -n tcp 6030 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` +done From 945f3ac25ec4888b0978d5fab403ed0ad7aa379a Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Thu, 9 Apr 2020 23:31:52 +0800 Subject: [PATCH 21/78] NOT_READY is not processed in a right way --- src/rpc/src/rpcMain.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index a241a8ce07..b9602696d6 100755 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -725,10 +725,6 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { return TSDB_CODE_INVALID_RESPONSE_TYPE; } - if (pHead->code == TSDB_CODE_NOT_READY) { - return TSDB_CODE_ALREADY_PROCESSED; - } - taosTmrStopA(&pConn->pTimer); pConn->retry = 0; @@ -935,8 +931,8 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { tTrace("%s %p, redirect is received, numOfIps:%d", pRpc->label, pConn, pContext->ipSet.numOfIps); rpcSendReqToServer(pRpc, pContext); } else if (pHead->code == TSDB_CODE_NOT_READY) { - pConn->pContext->code = pHead->code; - rpcProcessConnError(pConn->pContext, NULL); + pContext->code = pHead->code; + rpcProcessConnError(pContext, NULL); } else { rpcNotifyClient(pContext, &rpcMsg); } @@ -1101,7 +1097,7 @@ static void rpcProcessConnError(void *param, void *id) { tTrace("%s connection error happens", pRpc->label); - if ( pContext->numOfTry >= pContext->ipSet.numOfIps ) { + if ( pContext->numOfTry >= pContext->ipSet.numOfIps*2 ) { rpcMsg.msgType = pContext->msgType+1; rpcMsg.handle = pContext->ahandle; rpcMsg.code = pContext->code; From 358dad34ba4a2da1e47a2d030426e6be7d50be9a Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 10 Apr 2020 08:40:08 +0800 Subject: [PATCH 22/78] a minor change --- src/rpc/src/rpcMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index b9602696d6..73ea23cbf8 100755 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1097,7 +1097,7 @@ static void rpcProcessConnError(void *param, void *id) { tTrace("%s connection error happens", pRpc->label); - if ( pContext->numOfTry >= pContext->ipSet.numOfIps*2 ) { + if (pContext->numOfTry >= pContext->ipSet.numOfIps) { rpcMsg.msgType = pContext->msgType+1; rpcMsg.handle = pContext->ahandle; rpcMsg.code = pContext->code; From be2f915fb44e09af743d9895db2ef158bc5d7591 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 10 Apr 2020 08:59:17 +0800 Subject: [PATCH 23/78] option index is out of range --- src/vnode/main/src/vnodeMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index f5a780214b..7c92ad4f4f 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -344,7 +344,7 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { // TODO: this is a simple implement static int32_t vnodeReadCfg(SVnodeObj *pVnode) { - char option[3][16] = {0}; + char option[4][16] = {0}; char cfgFile[TSDB_FILENAME_LEN * 2] = {0}; sprintf(cfgFile, "%s/vnode%d/config", tsVnodeDir, pVnode->vgId); From 909876fa05f157bdbef59b172b1678297ec78891 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 10 Apr 2020 11:08:59 +0800 Subject: [PATCH 24/78] change errno index --- src/inc/taoserror.h | 138 ++++++++++++++++++++++---------------------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 0624af45c3..5cd38f6d6b 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -76,89 +76,89 @@ TAOS_DEFINE_ERROR(TSDB_CODE_NODE_OFFLINE, 0, 28, "node offline") TAOS_DEFINE_ERROR(TSDB_CODE_NETWORK_UNAVAIL, 0, 29, "network unavailable") // db & user -TAOS_DEFINE_ERROR(TSDB_CODE_DB_NOT_SELECTED, 0, 30, "db not selected") -TAOS_DEFINE_ERROR(TSDB_CODE_DB_ALREADY_EXIST, 0, 31, "database aleady exist") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_DB, 0, 32, "invalid database") -TAOS_DEFINE_ERROR(TSDB_CODE_MONITOR_DB_FORBIDDEN, 0, 33, "monitor db forbidden") -TAOS_DEFINE_ERROR(TSDB_CODE_USER_ALREADY_EXIST, 0, 34, "user already exist") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_USER, 0, 35, "invalid user") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PASS, 0, 36, "invalid password") +TAOS_DEFINE_ERROR(TSDB_CODE_DB_NOT_SELECTED, 0, 100, "db not selected") +TAOS_DEFINE_ERROR(TSDB_CODE_DB_ALREADY_EXIST, 0, 101, "database aleady exist") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_DB, 0, 102, "invalid database") +TAOS_DEFINE_ERROR(TSDB_CODE_MONITOR_DB_FORBIDDEN, 0, 103, "monitor db forbidden") +TAOS_DEFINE_ERROR(TSDB_CODE_USER_ALREADY_EXIST, 0, 104, "user already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_USER, 0, 105, "invalid user") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PASS, 0, 106, "invalid password") // table -TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ALREADY_EXIST, 0, 41, "table already exist") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE_ID, 0, 42, "invalid table id") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE_TYPE, 0, 43, "invalid table typee") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE, 0, 44, "invalid table name") -TAOS_DEFINE_ERROR(TSDB_CODE_NOT_SUPER_TABLE, 0, 45, "no super table") // operation only available for super table -TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_TABLE, 0, 46, "not active table") -TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ID_MISMATCH, 0, 47, "table id mismatch") +TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ALREADY_EXIST, 0, 200, "table already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE_ID, 0, 201, "invalid table id") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE_TYPE, 0, 202, "invalid table typee") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE, 0, 203, "invalid table name") +TAOS_DEFINE_ERROR(TSDB_CODE_NOT_SUPER_TABLE, 0, 204, "no super table") // operation only available for super table +TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_TABLE, 0, 205, "not active table") +TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ID_MISMATCH, 0, 206, "table id mismatch") // dnode & mnode -TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DNODES, 0, 50, "no enough dnodes") -TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ALREADY_EXIST, 0, 51, "dnode already exist") -TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_NOT_EXIST, 0, 52, "dnode not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_MASTER, 0, 53, "no master") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_REMOVE_MASTER, 0, 54, "no remove master") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_ID, 0, 55, "invalid query id") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_STREAM_ID, 0, 56, "invalid stream id") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CONNECTION, 0, 57, "invalid connection") -TAOS_DEFINE_ERROR(TSDB_CODE_SDB_ERROR, 0, 58, "sdb error") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DNODES, 0, 300, "no enough dnodes") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ALREADY_EXIST, 0, 301, "dnode already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_NOT_EXIST, 0, 302, "dnode not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_MASTER, 0, 303, "no master") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_REMOVE_MASTER, 0, 304, "no remove master") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_ID, 0, 305, "invalid query id") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_STREAM_ID, 0, 306, "invalid stream id") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CONNECTION, 0, 307, "invalid connection") +TAOS_DEFINE_ERROR(TSDB_CODE_SDB_ERROR, 0, 308, "sdb error") // acct -TAOS_DEFINE_ERROR(TSDB_CODE_ACCT_ALREADY_EXIST, 0, 60, "accounts already exist") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT, 0, 61, "invalid account") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT_PARAMETER, 0, 62, "invalid account parameter") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_ACCTS, 0, 63, "too many accounts") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_USERS, 0, 64, "too many users") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TABLES, 0, 65, "too many tables") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_DATABASES, 0, 66, "too many databases") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TIME_SERIES, 0, 67, "not enough time series") +TAOS_DEFINE_ERROR(TSDB_CODE_ACCT_ALREADY_EXIST, 0, 400, "accounts already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT, 0, 401, "invalid account") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT_PARAMETER, 0, 402, "invalid account parameter") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_ACCTS, 0, 403, "too many accounts") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_USERS, 0, 404, "too many users") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TABLES, 0, 405, "too many tables") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_DATABASES, 0, 406, "too many databases") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TIME_SERIES, 0, 407, "not enough time series") // grant -TAOS_DEFINE_ERROR(TSDB_CODE_AUTH_FAILURE, 0, 70, "auth failure") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_RIGHTS, 0, 71, "no rights") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_WRITE_ACCESS, 0, 72, "no write access") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_READ_ACCESS, 0, 73, "no read access") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 74, "grant expired") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED, 0, 75, "grant dnode limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_ACCT_LIMITED, 0, 76, "grant account limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TIMESERIES_LIMITED, 0, 77, "grant timeseries limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_LIMITED, 0, 78, "grant db limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_USER_LIMITED, 0, 79, "grant user limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CONN_LIMITED, 0, 80, "grant conn limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_LIMITED, 0, 81, "grant stream limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SPEED_LIMITED, 0, 82, "grant speed limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STORAGE_LIMITED, 0, 83, "grant storage limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED, 0, 84, "grant query time limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, 0, 85, "grant cpu limited") +TAOS_DEFINE_ERROR(TSDB_CODE_AUTH_FAILURE, 0, 400, "auth failure") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_RIGHTS, 0, 401, "no rights") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_WRITE_ACCESS, 0, 402, "no write access") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_READ_ACCESS, 0, 403, "no read access") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 404, "grant expired") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED, 0, 405, "grant dnode limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_ACCT_LIMITED, 0, 406, "grant account limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TIMESERIES_LIMITED, 0, 407, "grant timeseries limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_LIMITED, 0, 408, "grant db limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_USER_LIMITED, 0, 409, "grant user limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CONN_LIMITED, 0, 410, "grant conn limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_LIMITED, 0, 411, "grant stream limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SPEED_LIMITED, 0, 412, "grant speed limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STORAGE_LIMITED, 0, 413, "grant storage limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED, 0, 414, "grant query time limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, 0, 415, "grant cpu limited") // server -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VGROUP_ID, 0, 90, "invalid vgroup id") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VNODE_ID, 0, 91, "invalid vnode id") -TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_VNODE, 0, 92, "not active vnode") -TAOS_DEFINE_ERROR(TSDB_CODE_VG_INIT_FAILED, 0, 93, "vg init failed") -TAOS_DEFINE_ERROR(TSDB_CODE_SERV_NO_DISKSPACE, 0, 94, "server no diskspace") -TAOS_DEFINE_ERROR(TSDB_CODE_SERV_OUT_OF_MEMORY, 0, 95, "server out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISK_PERMISSIONS, 0, 96, "no disk permissions") -TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, 0, 97, "file corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, 0, 98, "memory corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VGROUP_ID, 0, 500, "invalid vgroup id") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VNODE_ID, 0, 501, "invalid vnode id") +TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_VNODE, 0, 502, "not active vnode") +TAOS_DEFINE_ERROR(TSDB_CODE_VG_INIT_FAILED, 0, 503, "vg init failed") +TAOS_DEFINE_ERROR(TSDB_CODE_SERV_NO_DISKSPACE, 0, 504, "server no diskspace") +TAOS_DEFINE_ERROR(TSDB_CODE_SERV_OUT_OF_MEMORY, 0, 505, "server out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISK_PERMISSIONS, 0, 506, "no disk permissions") +TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, 0, 507, "file corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, 0, 508, "memory corrupted") // client -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CLIENT_VERSION, 0, 101, "invalid client version") -TAOS_DEFINE_ERROR(TSDB_CODE_CLI_OUT_OF_MEMORY, 0, 102, "client out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_CLI_NO_DISKSPACE, 0, 103, "client no disk space") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIME_STAMP, 0, 104, "invalid timestamp") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_SQL, 0, 105, "invalid sql") -TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CACHE_ERASED, 0, 106, "query cache erased") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_MSG, 0, 107, "invalid query message") // failed to validate the sql expression msg by vnode -TAOS_DEFINE_ERROR(TSDB_CODE_SORTED_RES_TOO_MANY, 0, 108, "sorted res too many") // too many result for ordered super table projection query -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QHANDLE, 0, 109, "invalid handle") -TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CANCELLED, 0, 110, "query cancelled") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_IE, 0, 111, "invalid ie") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VALUE, 0, 112, "invalid value") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CLIENT_VERSION, 0, 601, "invalid client version") +TAOS_DEFINE_ERROR(TSDB_CODE_CLI_OUT_OF_MEMORY, 0, 602, "client out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_CLI_NO_DISKSPACE, 0, 603, "client no disk space") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIME_STAMP, 0, 604, "invalid timestamp") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_SQL, 0, 605, "invalid sql") +TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CACHE_ERASED, 0, 606, "query cache erased") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_MSG, 0, 607, "invalid query message") // failed to validate the sql expression msg by vnode +TAOS_DEFINE_ERROR(TSDB_CODE_SORTED_RES_TOO_MANY, 0, 608, "sorted res too many") // too many result for ordered super table projection query +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QHANDLE, 0, 609, "invalid handle") +TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CANCELLED, 0, 610, "query cancelled") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_IE, 0, 611, "invalid ie") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VALUE, 0, 612, "invalid value") // others -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_FILE_FORMAT, 0, 120, "invalid file format") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_FILE_FORMAT, 0, 700, "invalid file format") #ifdef TAOS_ERROR_C From b17e7329388899107763295102dc965269c2b298 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 10 Apr 2020 11:40:46 +0800 Subject: [PATCH 25/78] make output clear and add general/db testcases. --- .travis.yml | 13 +++++++++---- tests/script/basicSuite.sim | 7 ++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5cff3bc72b..63c62d1a8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,14 +51,19 @@ script: case $TRAVIS_OS_NAME in linux) cd ../tests/script - sudo ./test.sh 2>&1 | tee out.txt - cat out.txt - grep success out.txt + sudo ./test.sh 2>&1 | grep 'success\|failed' | tee out.txt + total_success=`grep success out.txt | wc -l` + + if [ "$total_success" -gt "0" ]; then + total_success=`expr $total_success - 1` + fi + echo "Total $total_success success" - grep failed out.txt + total_failed=`grep failed out.txt | wc -l` echo "Total $total_failed failed" + if [ "$total_failed" -ne "0" ]; then exit $total_failed fi diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index 290472b15d..440ec5592b 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -1,8 +1,13 @@ ################################# - run general/table/basic1.sim run general/table/basic2.sim run general/table/basic3.sim +run general/db/basic1.sim +run general/db/basic2.sim +run general/db/basic3.sim +run general/db/basic4.sim +run general/db/basic5.sim + ################################## From 0bb86882a5a5682291028c07cb3932924a6fbac1 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 10 Apr 2020 11:48:17 +0800 Subject: [PATCH 26/78] Update administrator-ch.md --- documentation/webdocs/markdowndocs/administrator-ch.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/documentation/webdocs/markdowndocs/administrator-ch.md b/documentation/webdocs/markdowndocs/administrator-ch.md index 35beb610f2..cbccc03cbb 100644 --- a/documentation/webdocs/markdowndocs/administrator-ch.md +++ b/documentation/webdocs/markdowndocs/administrator-ch.md @@ -102,6 +102,11 @@ RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接 日志文件目录，客户端和服务器的运行日志将写入该目录。 +**shellActivityTimer** +- 默认值：3 + +系统在服务端保持结果集的最长时间，范围[1-120]。 + **maxUsers** - 默认值：10,000 @@ -411,4 +416,4 @@ TDengine启动后，会自动创建一个监测数据库`LOG`，并自动将服 这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项`monitor`将其关闭或打开。 -[1]: https://github.com/taosdata/TDengine/tree/develop/importSampleData \ No newline at end of file +[1]: https://github.com/taosdata/TDengine/tree/develop/importSampleData From 4ecb5e0f7fd29d2e9e70efe6b623bc1ecf75de40 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 10 Apr 2020 11:51:00 +0800 Subject: [PATCH 27/78] refCount may be negative is vnodeRelease is called appropriately add assert there in case it happens --- src/vnode/main/src/vnodeMain.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 182b4b6257..8d36f59da0 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -202,11 +202,13 @@ int32_t vnodeClose(int32_t vgId) { void vnodeRelease(void *pVnodeRaw) { SVnodeObj *pVnode = pVnodeRaw; + int32_t vgId = pVnode->vgId; int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); + assert(refCount >= 0); if (refCount > 0) { - dTrace("pVnode:%p vgId:%d, release vnode, refCount:%d", pVnode, pVnode->vgId, refCount); + dTrace("pVnode:%p vgId:%d, release vnode, refCount:%d", pVnode, vgId, refCount); return; } From 8c86feba0a60440419789fac1d3f66404de51e91 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 10 Apr 2020 12:21:56 +0800 Subject: [PATCH 28/78] fix print typo in general/db/basic5.sim. --- tests/script/general/db/basic5.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/general/db/basic5.sim b/tests/script/general/db/basic5.sim index ebb7d3c967..55a42899ec 100644 --- a/tests/script/general/db/basic5.sim +++ b/tests/script/general/db/basic5.sim @@ -46,7 +46,7 @@ if $data02 != ready then return -1 endi -print =============== drop table +print =============== drop database sql drop database d1 sql show databases From ed2254dbcc13a1e578fba06ce371549358c679c3 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 10 Apr 2020 14:09:50 +0800 Subject: [PATCH 29/78] [TD-114] let create table wait until master selected --- src/inc/mnode.h | 5 +++- src/inc/taosmsg.h | 1 - src/mnode/inc/mgmtShell.h | 1 + src/mnode/inc/mgmtVgroup.h | 1 + src/mnode/src/mgmtDnode.c | 13 +++++---- src/mnode/src/mgmtProfile.c | 2 ++ src/mnode/src/mgmtShell.c | 9 +++++++ src/mnode/src/mgmtTable.c | 47 ++++++++++++++++++++++----------- src/mnode/src/mgmtVgroup.c | 13 +++++++++ src/vnode/main/src/vnodeMain.c | 1 - src/vnode/main/src/vnodeWrite.c | 3 ++- tests/script/tmp/prepare.sim | 3 ++- 12 files changed, 73 insertions(+), 26 deletions(-) diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 903b172068..dec9292209 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -149,7 +149,8 @@ typedef struct _vg_obj { int32_t lbDnodeId; int32_t lbTime; int8_t status; - int8_t reserved[14]; + int8_t inUse; + int8_t reserved[13]; int8_t updateEnd[1]; int32_t refCount; struct _vg_obj *prev, *next; @@ -243,6 +244,8 @@ typedef struct { int8_t received; int8_t successed; int8_t expected; + int8_t retry; + int8_t maxRetry; int32_t contLen; int32_t code; void *ahandle; diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 5f07d6ec99..f5168a2c9e 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -510,7 +510,6 @@ typedef struct SRetrieveTableRsp { typedef struct { int32_t vgId; - int32_t vnode; int64_t totalStorage; int64_t compStorage; int64_t pointsWritten; diff --git a/src/mnode/inc/mgmtShell.h b/src/mnode/inc/mgmtShell.h index b92e9de1f4..171c93a390 100644 --- a/src/mnode/inc/mgmtShell.h +++ b/src/mnode/inc/mgmtShell.h @@ -31,6 +31,7 @@ void mgmtAddShellShowMetaHandle(uint8_t showType, SShowMetaFp fp); void mgmtAddShellShowRetrieveHandle(uint8_t showType, SShowRetrieveFp fp); void mgmtAddToShellQueue(SQueuedMsg *queuedMsg); +void mgmtDealyedAddToShellQueue(SQueuedMsg *queuedMsg); void mgmtSendSimpleResp(void *thandle, int32_t code); #ifdef __cplusplus diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h index 83e003e063..072c616f3d 100644 --- a/src/mnode/inc/mgmtVgroup.h +++ b/src/mnode/inc/mgmtVgroup.h @@ -37,6 +37,7 @@ void mgmtDropAllVgroups(SDbObj *pDropDb); void * mgmtGetNextVgroup(void *pNode, SVgObj **pVgroup); void mgmtUpdateVgroup(SVgObj *pVgroup); +void mgmtUpdateVgroupStatus(SVgObj *pVgroup, int32_t dnodeId, SVnodeLoad *pVload); void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb); void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle); diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index e4ab114090..2f4abc5cfe 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -189,18 +189,21 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { int32_t openVnodes = htons(pStatus->openVnodes); for (int32_t j = 0; j < openVnodes; ++j) { - pDnode->vload[j].vgId = htonl(pStatus->load[j].vgId); - pDnode->vload[j].totalStorage = htobe64(pStatus->load[j].totalStorage); - pDnode->vload[j].compStorage = htobe64(pStatus->load[j].compStorage); - pDnode->vload[j].pointsWritten = htobe64(pStatus->load[j].pointsWritten); + SVnodeLoad *pVload = &pStatus->load[j]; + pDnode->vload[j].vgId = htonl(pVload->vgId); + pDnode->vload[j].totalStorage = htobe64(pVload->totalStorage); + pDnode->vload[j].compStorage = htobe64(pVload->compStorage); + pDnode->vload[j].pointsWritten = htobe64(pVload->pointsWritten); SVgObj *pVgroup = mgmtGetVgroup(pDnode->vload[j].vgId); if (pVgroup == NULL) { SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->privateIp); mPrint("dnode:%d, vgroup:%d not exist in mnode, drop it", pDnode->dnodeId, pDnode->vload[j].vgId); mgmtSendDropVnodeMsg(pDnode->vload[j].vgId, &ipSet, NULL); + } else { + mgmtUpdateVgroupStatus(pVgroup, pDnode->dnodeId, pVload); + mgmtReleaseVgroup(pVgroup); } - mgmtReleaseVgroup(pVgroup); } if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { diff --git a/src/mnode/src/mgmtProfile.c b/src/mnode/src/mgmtProfile.c index 0360432971..2b22fae47a 100644 --- a/src/mnode/src/mgmtProfile.c +++ b/src/mnode/src/mgmtProfile.c @@ -806,6 +806,8 @@ void* mgmtCloneQueuedMsg(SQueuedMsg *pSrcMsg) { pDestMsg->msgType = pSrcMsg->msgType; pDestMsg->pCont = pSrcMsg->pCont; pDestMsg->contLen = pSrcMsg->contLen; + pDestMsg->retry = pSrcMsg->retry; + pDestMsg->maxRetry= pSrcMsg->maxRetry; pDestMsg->pUser = pSrcMsg->pUser; pDestMsg->usePublicIp = pSrcMsg->usePublicIp; diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index 7b6a2654ae..880c6d0c10 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -128,6 +128,15 @@ void mgmtAddToShellQueue(SQueuedMsg *queuedMsg) { taosScheduleTask(tsMgmtTranQhandle, &schedMsg); } +static void mgmtDoDealyedAddToShellQueue(void *param, void *tmrId) { + mgmtAddToShellQueue(param); +} + +void mgmtDealyedAddToShellQueue(SQueuedMsg *queuedMsg) { + void *unUsed = NULL; + taosTmrReset(mgmtDoDealyedAddToShellQueue, 1000, queuedMsg, tsMgmtTmr, &unUsed); +} + static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { if (rpcMsg == NULL || rpcMsg->pCont == NULL) { return; diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index a14bdd058f..b536bb5ac9 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -540,7 +540,7 @@ static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { SCMCreateTableMsg *pCreate = pMsg->pCont; pMsg->pTable = mgmtGetTable(pCreate->tableId); - if (pMsg->pTable != NULL) { + if (pMsg->pTable != NULL && pMsg->retry == 0) { if (pCreate->igExists) { mTrace("table:%s, is already exist", pCreate->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS); @@ -1300,7 +1300,11 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) { return; } - pMsg->pTable = (STableObj *)mgmtDoCreateChildTable(pCreate, pVgroup, sid); + if (pMsg->retry == 0) { + pMsg->pTable = (STableObj *)mgmtDoCreateChildTable(pCreate, pVgroup, sid); + } else { + pMsg->pTable = mgmtGetTable(pCreate->tableId); + } if (pMsg->pTable == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); return; @@ -1315,6 +1319,7 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) { SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup); SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg); newMsg->ahandle = pMsg->pTable; + newMsg->maxRetry = 5; mgmtIncTableRef(pMsg->pTable); SRpcMsg rpcMsg = { .handle = newMsg, @@ -1737,30 +1742,40 @@ static void mgmtProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { queueMsg->received++; SChildTableObj *pTable = queueMsg->ahandle; - mTrace("table:%s, create table rsp received, thandle:%p ahandle:%p result:%s", pTable->info.tableId, queueMsg->thandle, - rpcMsg->handle, tstrerror(rpcMsg->code)); + mTrace("table:%s, create table rsp received, thandle:%p result:%s", pTable->info.tableId, queueMsg->thandle, + tstrerror(rpcMsg->code)); if (rpcMsg->code != TSDB_CODE_SUCCESS) { - SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, - .table = tsChildTableSdb, - .pObj = pTable - }; - sdbDeleteRow(&oper); + if (queueMsg->retry++ < queueMsg->maxRetry) { + mTrace("table:%s, create table rsp received, retry:%d thandle:%p result:%s", pTable->info.tableId, + queueMsg->retry, queueMsg->thandle, tstrerror(rpcMsg->code)); + mgmtDealyedAddToShellQueue(queueMsg); + } else { + mError("table:%s, failed to create in dnode, thandle:%p result:%s", pTable->info.tableId, + queueMsg->thandle, tstrerror(rpcMsg->code)); + + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsChildTableSdb, + .pObj = pTable + }; + sdbDeleteRow(&oper); - mError("table:%s, failed to create in dnode, reason:%s", pTable->info.tableId, tstrerror(rpcMsg->code)); - mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); + mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); + mgmtFreeQueuedMsg(queueMsg); + } } else { - mTrace("table:%s, created in dnode", pTable->info.tableId); + mTrace("table:%s, created in dnode, thandle:%p result:%s", pTable->info.tableId, queueMsg->thandle, + tstrerror(rpcMsg->code)); + if (queueMsg->msgType != TSDB_MSG_TYPE_CM_CREATE_TABLE) { mTrace("table:%s, start to get meta", pTable->info.tableId); - mgmtAddToShellQueue(mgmtCloneQueuedMsg(queueMsg)); + mgmtAddToShellQueue(queueMsg); } else { mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); + mgmtFreeQueuedMsg(queueMsg); } } - - mgmtFreeQueuedMsg(queueMsg); } // not implemented yet diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 4c969124a0..6b27fbbc83 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -18,6 +18,7 @@ #include "taoserror.h" #include "tlog.h" #include "tbalance.h" +#include "tsync.h" #include "tcluster.h" #include "mnode.h" #include "mgmtDb.h" @@ -209,6 +210,18 @@ void mgmtUpdateVgroup(SVgObj *pVgroup) { mgmtSendCreateVgroupMsg(pVgroup, NULL); } +void mgmtUpdateVgroupStatus(SVgObj *pVgroup, int32_t dnodeId, SVnodeLoad *pVload) { + if (pVload->role == TAOS_SYNC_ROLE_MASTER) { + for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; + if (pVgid->dnodeId == dnodeId) { + pVgroup->inUse = i; + break; + } + } + } +} + SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb) { return pDb->pHead; } diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 182b4b6257..8db00ed1b1 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -285,7 +285,6 @@ static void vnodeBuildVloadMsg(char *pNode, void * param) { SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++]; pLoad->vgId = htonl(pVnode->vgId); - pLoad->vnode = htonl(pVnode->vgId); pLoad->status = pVnode->status; pLoad->role = pVnode->role; } diff --git a/src/vnode/main/src/vnodeWrite.c b/src/vnode/main/src/vnodeWrite.c index 5e03305487..6f06585fa9 100644 --- a/src/vnode/main/src/vnodeWrite.c +++ b/src/vnode/main/src/vnodeWrite.c @@ -57,7 +57,8 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { if (pVnode->status != TAOS_VN_STATUS_READY) return TSDB_CODE_NOT_ACTIVE_VNODE; - // if (pVnode->replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) + if (pVnode->syncCfg.replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) + return TSDB_CODE_NO_MASTER; // assign version pVnode->version++; diff --git a/tests/script/tmp/prepare.sim b/tests/script/tmp/prepare.sim index 1f0b893e6d..731b707434 100644 --- a/tests/script/tmp/prepare.sim +++ b/tests/script/tmp/prepare.sim @@ -1,3 +1,4 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2 \ No newline at end of file +system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2 +system sh/deploy.sh -n dnode3 -m 192.168.0.1 -i 192.168.0.3 \ No newline at end of file From b723946960c88ce5cb4a7be2a778a14ea0b7d08f Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 10 Apr 2020 14:23:25 +0800 Subject: [PATCH 30/78] [TD-114] let replica 3 can create table --- src/vnode/main/src/vnodeWrite.c | 92 ++++++++++++++------------------- 1 file changed, 39 insertions(+), 53 deletions(-) diff --git a/src/vnode/main/src/vnodeWrite.c b/src/vnode/main/src/vnodeWrite.c index 6f06585fa9..1692bc14eb 100644 --- a/src/vnode/main/src/vnodeWrite.c +++ b/src/vnode/main/src/vnodeWrite.c @@ -110,36 +110,27 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe int32_t code = 0; dTrace("pVnode:%p vgId:%d, table:%s, start to create", pVnode, pVnode->vgId, pTable->tableId); - pTable->numOfColumns = htons(pTable->numOfColumns); - pTable->numOfTags = htons(pTable->numOfTags); - pTable->sid = htonl(pTable->sid); - pTable->sversion = htonl(pTable->sversion); - pTable->tagDataLen = htonl(pTable->tagDataLen); - pTable->sqlDataLen = htonl(pTable->sqlDataLen); - pTable->uid = htobe64(pTable->uid); - pTable->superTableUid = htobe64(pTable->superTableUid); - pTable->createdTime = htobe64(pTable->createdTime); + int16_t numOfColumns = htons(pTable->numOfColumns); + int16_t numOfTags = htons(pTable->numOfTags); + int32_t sid = htonl(pTable->sid); + uint64_t uid = htobe64(pTable->uid); SSchema *pSchema = (SSchema *) pTable->data; - int totalCols = pTable->numOfColumns + pTable->numOfTags; - for (int i = 0; i < totalCols; i++) { - pSchema[i].colId = htons(pSchema[i].colId); - pSchema[i].bytes = htons(pSchema[i].bytes); - } - + int32_t totalCols = numOfColumns + numOfTags; + STableCfg tCfg; - tsdbInitTableCfg(&tCfg, pTable->tableType, pTable->uid, pTable->sid); + tsdbInitTableCfg(&tCfg, pTable->tableType, uid, sid); - STSchema *pDestSchema = tdNewSchema(pTable->numOfColumns); - for (int i = 0; i < pTable->numOfColumns; i++) { - tdSchemaAppendCol(pDestSchema, pSchema[i].type, pSchema[i].colId, pSchema[i].bytes); + STSchema *pDestSchema = tdNewSchema(numOfColumns); + for (int i = 0; i < numOfColumns; i++) { + tdSchemaAppendCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)); } tsdbTableSetSchema(&tCfg, pDestSchema, false); - if (pTable->numOfTags != 0) { - STSchema *pDestTagSchema = tdNewSchema(pTable->numOfTags); - for (int i = pTable->numOfColumns; i < totalCols; i++) { - tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, pSchema[i].colId, pSchema[i].bytes); + if (numOfTags != 0) { + STSchema *pDestTagSchema = tdNewSchema(numOfTags); + for (int i = numOfColumns; i < totalCols; i++) { + tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)); } tsdbTableSetTagSchema(&tCfg, pDestTagSchema, false); @@ -147,9 +138,9 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe int accumBytes = 0; SDataRow dataRow = tdNewDataRowFromSchema(pDestTagSchema); - for (int i = 0; i < pTable->numOfTags; i++) { + for (int i = 0; i < numOfTags; i++) { tdAppendColVal(dataRow, pTagData + accumBytes, pDestTagSchema->columns + i); - accumBytes += pSchema[i + pTable->numOfColumns].bytes; + accumBytes += htons(pSchema[i + numOfColumns].bytes); } tsdbTableSetTagValue(&tCfg, dataRow, false); } @@ -183,51 +174,46 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet int32_t code = 0; dTrace("pVnode:%p vgId:%d, table:%s, start to alter", pVnode, pVnode->vgId, pTable->tableId); - pTable->numOfColumns = htons(pTable->numOfColumns); - pTable->numOfTags = htons(pTable->numOfTags); - pTable->sid = htonl(pTable->sid); - pTable->sversion = htonl(pTable->sversion); - pTable->tagDataLen = htonl(pTable->tagDataLen); - pTable->sqlDataLen = htonl(pTable->sqlDataLen); - pTable->uid = htobe64(pTable->uid); - pTable->superTableUid = htobe64(pTable->superTableUid); - pTable->createdTime = htobe64(pTable->createdTime); + int16_t numOfColumns = htons(pTable->numOfColumns); + int16_t numOfTags = htons(pTable->numOfTags); + int32_t sid = htonl(pTable->sid); + uint64_t uid = htobe64(pTable->uid); SSchema *pSchema = (SSchema *) pTable->data; - int totalCols = pTable->numOfColumns + pTable->numOfTags; - for (int i = 0; i < totalCols; i++) { - pSchema[i].colId = htons(pSchema[i].colId); - pSchema[i].bytes = htons(pSchema[i].bytes); - } - + int32_t totalCols = numOfColumns + numOfTags; + STableCfg tCfg; - tsdbInitTableCfg(&tCfg, pTable->tableType, pTable->uid, pTable->sid); + tsdbInitTableCfg(&tCfg, pTable->tableType, uid, sid); - STSchema *pDestSchema = tdNewSchema(pTable->numOfColumns); - for (int i = 0; i < pTable->numOfColumns; i++) { - tdSchemaAppendCol(pDestSchema, pSchema[i].type, pSchema[i].colId, pSchema[i].bytes); + STSchema *pDestSchema = tdNewSchema(numOfColumns); + for (int i = 0; i < numOfColumns; i++) { + tdSchemaAppendCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)); } tsdbTableSetSchema(&tCfg, pDestSchema, false); - if (pTable->numOfTags != 0) { - STSchema *pDestTagSchema = tdNewSchema(pTable->numOfTags); - for (int i = pTable->numOfColumns; i < totalCols; i++) { - tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, pSchema[i].colId, pSchema[i].bytes); + if (numOfTags != 0) { + STSchema *pDestTagSchema = tdNewSchema(numOfTags); + for (int i = numOfColumns; i < totalCols; i++) { + tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)); } - tsdbTableSetSchema(&tCfg, pDestTagSchema, false); + tsdbTableSetTagSchema(&tCfg, pDestTagSchema, false); char *pTagData = pTable->data + totalCols * sizeof(SSchema); int accumBytes = 0; SDataRow dataRow = tdNewDataRowFromSchema(pDestTagSchema); - for (int i = 0; i < pTable->numOfTags; i++) { + for (int i = 0; i < numOfTags; i++) { tdAppendColVal(dataRow, pTagData + accumBytes, pDestTagSchema->columns + i); - accumBytes += pSchema[i + pTable->numOfColumns].bytes; + accumBytes += htons(pSchema[i + numOfColumns].bytes); } tsdbTableSetTagValue(&tCfg, dataRow, false); } - code = tsdbAlterTable(pVnode->tsdb, &tCfg); + void *pTsdb = vnodeGetTsdb(pVnode); + code = tsdbAlterTable(pTsdb, &tCfg); + + tfree(pDestSchema); + dTrace("pVnode:%p vgId:%d, table:%s, alter table result:%d", pVnode, pVnode->vgId, pTable->tableId, code); return code; @@ -238,7 +224,7 @@ static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet int32_t code = 0; dTrace("pVnode:%p vgId:%d, stable:%s, start to drop", pVnode, pVnode->vgId, pTable->tableId); - pTable->uid = htobe64(pTable->uid); + // int64_t uid = htobe64(pTable->uid); // TODO: drop stable in vvnode //void *pTsdb = dnodeGetVnodeTsdb(pMsg->pVnode); From f843a22bb81ef9ca5f2e5250f1cefd5d59e5c210 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 10 Apr 2020 15:05:33 +0800 Subject: [PATCH 31/78] [td-98]fix uninitialised value access warning by valgrind --- src/util/src/tglobalcfg.c | 69 +++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index 8a0d66068e..69178e35e9 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -14,13 +14,12 @@ */ #include "os.h" -#include +#include "taosdef.h" +#include "taoserror.h" #include "tglobalcfg.h" #include "tkey.h" #include "tlog.h" -#include "taosdef.h" -#include "taoserror.h" #include "tsocket.h" #include "tsystem.h" #include "tutil.h" @@ -833,9 +832,8 @@ void tsReadGlobalLogConfig() { FILE * fp; char * line, *option, *value; - size_t len; int olen, vlen; - char fileName[128]; + char fileName[PATH_MAX] = {0}; mdebugFlag = 135; sdbDebugFlag = 135; @@ -851,27 +849,30 @@ void tsReadGlobalLogConfig() { wordfree(&full_path); tsReadLogOption("logDir", logDir); + sprintf(fileName, "%s/taos.cfg", configDir); fp = fopen(fileName, "r"); if (fp == NULL) { - printf("\noption file:%s not found, all options are set to system default\n", fileName); + printf("\nconfig file:%s not found, all variables are set to default\n", fileName); return; } - - line = NULL; + + size_t len = 1024; + line = calloc(1, len); + while (!feof(fp)) { - tfree(line); - line = option = value = NULL; - len = olen = vlen = 0; + memset(line, 0, len); + + option = value = NULL; + olen = vlen = 0; getline(&line, &len, fp); - if (line == NULL) break; - paGetToken(line, &option, &olen); + paGetToken(line, len, &option, &olen); if (olen == 0) continue; option[olen] = 0; - paGetToken(option + olen + 1, &value, &vlen); + paGetToken(option + olen + 1, len, &value, &vlen); if (vlen == 0) continue; value[vlen] = 0; @@ -885,44 +886,45 @@ void tsReadGlobalLogConfig() { bool tsReadGlobalConfig() { tsInitGlobalConfig(); - FILE * fp; char * line, *option, *value, *value1; - size_t len; int olen, vlen, vlen1; - char fileName[128]; + char fileName[PATH_MAX] = {0}; sprintf(fileName, "%s/taos.cfg", configDir); - fp = fopen(fileName, "r"); - if (fp == NULL) { - } else { - line = NULL; + FILE* fp = fopen(fileName, "r"); + + size_t len = 1024; + line = calloc(1, len); + + if (fp != NULL) { while (!feof(fp)) { - tfree(line); - line = option = value = NULL; - len = olen = vlen = 0; + memset(line, 0, len); + + option = value = NULL; + olen = vlen = 0; getline(&line, &len, fp); - if (line == NULL) break; - paGetToken(line, &option, &olen); + paGetToken(line, len, &option, &olen); if (olen == 0) continue; option[olen] = 0; - paGetToken(option + olen + 1, &value, &vlen); + paGetToken(option + olen + 1, len, &value, &vlen); if (vlen == 0) continue; value[vlen] = 0; // For dataDir, the format is: // dataDir /mnt/disk1 0 - paGetToken(value + vlen + 1, &value1, &vlen1); - + paGetToken(value + vlen + 1, len, &value1, &vlen1); + tsReadConfigOption(option, value); } - tfree(line); fclose(fp); } + tfree(line); + if (tsReadStorageConfig) { tsReadStorageConfig(); } @@ -976,6 +978,7 @@ bool tsReadGlobalConfig() { strcpy(tsLocalIp, tsPrivateIp); } + // todo refactor tsVersion = 0; for (int i = 0; i < 10; i++) { if (version[i] >= '0' && version[i] <= '9') { @@ -984,6 +987,7 @@ bool tsReadGlobalConfig() { break; } } + tsVersion = 10 * tsVersion; return true; @@ -994,10 +998,11 @@ int tsCfgDynamicOptions(char *msg) { int olen, vlen, code = 0; int vint = 0; - paGetToken(msg, &option, &olen); + size_t len = 120; + paGetToken(msg, len, &option, &olen); if (olen == 0) return TSDB_CODE_INVALID_MSG_CONTENT; - paGetToken(option + olen + 1, &value, &vlen); + paGetToken(option + olen + 1, len, &value, &vlen); if (vlen == 0) vint = 135; else { From 1da0f92d841fbeb76ded3be11753bbb8e1bf1eb3 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 10 Apr 2020 15:06:20 +0800 Subject: [PATCH 32/78] [td-98] add clone function for array --- src/util/inc/tarray.h | 6 ++++++ src/util/src/tarray.c | 15 +++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 6b70780d10..28ebb18235 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -100,6 +100,12 @@ void taosArrayRemove(SArray* pArray, size_t index); */ void taosArrayCopy(SArray* pDst, SArray* pSrc); +/** + * clone a new array + * @param pSrc + */ +SArray* taosArrayClone(SArray* pSrc); + /** * destroy array list * @param pArray diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index eb1d213317..d97b220a40 100755 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -160,6 +160,21 @@ void taosArrayCopy(SArray* pDst, SArray* pSrc) { pDst->size = pSrc->size; } +SArray* taosArrayClone(SArray* pSrc) { + assert(pSrc != NULL); + + if (pSrc->size == 0) { // empty array list + return taosArrayInit(8, pSrc->elemSize); + } + + SArray* dst = taosArrayInit(pSrc->size, pSrc->elemSize); + + memcpy(dst->pData, pSrc->pData, pSrc->elemSize * pSrc->size); + dst->size = pSrc->size; + return dst; +} + + void taosArrayDestroy(SArray* pArray) { if (pArray == NULL) { return; From 66d6800315a09378755a818bf42c1817b4678064 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 10 Apr 2020 15:07:23 +0800 Subject: [PATCH 33/78] [td-98] add bi-direction iterator for skiplist --- src/util/inc/tskiplist.h | 24 +- src/util/inc/tutil.h | 2 +- src/util/src/tskiplist.c | 395 +++++++++++--------------------- src/util/src/tutil.c | 7 +- src/util/tests/skiplistTest.cpp | 1 + 5 files changed, 160 insertions(+), 269 deletions(-) diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index 18404f89c2..dc67d6dad5 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TBASE_TSKIPLIST_H -#define TBASE_TSKIPLIST_H +#ifndef TDENGINE_TSKIPLIST_H +#define TDENGINE_TSKIPLIST_H #ifdef __cplusplus extern "C" { @@ -24,10 +24,6 @@ extern "C" { #include "taosdef.h" #include "tarray.h" -/* - * key of each node - * todo move to as the global structure in all search codes... - */ #define MAX_SKIP_LIST_LEVEL 15 #define SKIP_LIST_RECORD_PERFORMANCE 0 @@ -35,11 +31,11 @@ typedef char *SSkipListKey; typedef char *(*__sl_key_fn_t)(const void *); /** + * the skip list node is located in a consecutive memory area, * the format of skip list node is as follows: * +------------+-----------------------+------------------------+-----+------+ * | node level | forward pointer array | backward pointer array | key | data | * +------------+-----------------------+------------------------+-----+------+ - * the skiplist node is located in a consecutive memory area, key will not be copy to skip list */ typedef struct SSkipListNode { uint8_t level; @@ -136,7 +132,8 @@ typedef struct SSkipList { typedef struct SSkipListIterator { SSkipList * pSkipList; SSkipListNode *cur; - int64_t num; + int32_t step; // the number of nodes that have been checked already + int32_t order; // order of the iterator } SSkipListIterator; /** @@ -205,6 +202,15 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); */ SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); +/** + * create skip list iterator from the given node and specified the order + * @param pSkipList + * @param pNode start position, instead of the first node in skip list + * @param order traverse order of the iterator + * @return + */ +SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order); + /** * forward the skip list iterator * @param iter @@ -245,4 +251,4 @@ void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); } #endif -#endif // TBASE_TSKIPLIST_H +#endif // TDENGINE_TSKIPLIST_H diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index b80aad1ceb..b7877b4307 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -125,7 +125,7 @@ char* strreplace(const char* str, const char* pattern, const char* rep); int32_t strdequote(char *src); -char *paGetToken(char *src, char **token, int32_t *tokenLen); +char *paGetToken(char *src, size_t maxLen, char **token, int32_t *tokenLen); void taosMsleep(int32_t mseconds); diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 3af7bfd2f2..40295fa1ea 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -17,6 +17,7 @@ #include "tlog.h" #include "tskiplist.h" #include "tutil.h" +#include "tcompare.h" static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, int32_t level) { // record link count in each level #if SKIP_LIST_RECORD_PERFORMANCE @@ -72,196 +73,54 @@ memset(pNode, 0, SL_NODE_HEADER_SIZE(_l));\ static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode); static SSkipListNode* tSkipListDoAppend(SSkipList *pSkipList, SSkipListNode *pNode); +static SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); -int32_t compareInt32Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT32_VAL(pLeft) - GET_INT32_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -int32_t compareInt64Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT64_VAL(pLeft) - GET_INT64_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -int32_t compareInt16Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT16_VAL(pLeft) - GET_INT16_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -int32_t compareInt8Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT8_VAL(pLeft) - GET_INT8_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -int32_t compareIntDoubleVal(const void *pLeft, const void *pRight) { - // int64_t lhs = ((SSkipListKey *)pLeft)->i64Key; - // double rhs = ((SSkipListKey *)pRight)->dKey; - // if (fabs(lhs - rhs) < FLT_EPSILON) { - // return 0; - // } else { - // return (lhs > rhs) ? 1 : -1; - // } - return 0; -} - -int32_t compareDoubleIntVal(const void *pLeft, const void *pRight) { - // double lhs = ((SSkipListKey *)pLeft)->dKey; - // int64_t rhs = ((SSkipListKey *)pRight)->i64Key; - // if (fabs(lhs - rhs) < FLT_EPSILON) { - // return 0; - // } else { - // return (lhs > rhs) ? 1 : -1; - // } - return 0; -} - -int32_t compareDoubleVal(const void *pLeft, const void *pRight) { - double ret = GET_DOUBLE_VAL(pLeft) - GET_DOUBLE_VAL(pRight); - if (fabs(ret) < FLT_EPSILON) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -int32_t compareStrVal(const void *pLeft, const void *pRight) { - int32_t ret = strcmp(pLeft, pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -int32_t compareWStrVal(const void *pLeft, const void *pRight) { - // SSkipListKey *pL = (SSkipListKey *)pLeft; - // SSkipListKey *pR = (SSkipListKey *)pRight; - // - // if (pL->nLen == 0 && pR->nLen == 0) { - // return 0; - // } - // - // // handle only one-side bound compare situation, there is only lower bound or only upper bound - // if (pL->nLen == -1) { - // return 1; // no lower bound, lower bound is minimum, always return -1; - // } else if (pR->nLen == -1) { - // return -1; // no upper bound, upper bound is maximum situation, always return 1; - // } - // - // int32_t ret = wcscmp(((SSkipListKey *)pLeft)->wpz, ((SSkipListKey *)pRight)->wpz); - // - // if (ret == 0) { - // return 0; - // } else { - // return ret > 0 ? 1 : -1; - // } - return 0; -} - -static __compar_fn_t getKeyFilterComparator(SSkipList *pSkipList, int32_t filterDataType) { - __compar_fn_t comparFn = NULL; - - switch (pSkipList->keyInfo.type) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: { - if (filterDataType == TSDB_DATA_TYPE_BIGINT) { - comparFn = compareInt64Val; - break; - } - } - case TSDB_DATA_TYPE_BOOL: { - if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { - comparFn = compareInt32Val; - } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { - comparFn = compareIntDoubleVal; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: { +//static __compar_fn_t getComparFunc(SSkipList *pSkipList, int32_t filterDataType) { +// __compar_fn_t comparFn = NULL; +// +// switch (pSkipList->keyInfo.type) { +// case TSDB_DATA_TYPE_TINYINT: +// case TSDB_DATA_TYPE_SMALLINT: +// case TSDB_DATA_TYPE_INT: +// case TSDB_DATA_TYPE_BIGINT: { +// if (filterDataType == TSDB_DATA_TYPE_BIGINT) { +// comparFn = compareInt64Val; +// break; +// } +// } +// case TSDB_DATA_TYPE_BOOL: { // if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { -// comparFn = compareDoubleIntVal; +// comparFn = compareInt32Val; // } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { +// comparFn = compareIntDoubleVal; +// } +// break; +// } +// case TSDB_DATA_TYPE_FLOAT: +// case TSDB_DATA_TYPE_DOUBLE: { +//// if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { +//// comparFn = compareDoubleIntVal; +//// } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { +//// comparFn = compareDoubleVal; +//// } +// if (filterDataType == TSDB_DATA_TYPE_DOUBLE) { // comparFn = compareDoubleVal; // } - if (filterDataType == TSDB_DATA_TYPE_DOUBLE) { - comparFn = compareDoubleVal; - } - break; - } - case TSDB_DATA_TYPE_BINARY: - comparFn = compareStrVal; - break; - case TSDB_DATA_TYPE_NCHAR: - comparFn = compareWStrVal; - break; - default: - comparFn = compareInt32Val; - break; - } - - return comparFn; -} - -static __compar_fn_t getKeyComparator(int32_t keyType) { - __compar_fn_t comparFn = NULL; - - switch (keyType) { - case TSDB_DATA_TYPE_TINYINT: - comparFn = compareInt8Val; - break; - case TSDB_DATA_TYPE_SMALLINT: - comparFn = compareInt16Val; - break; - case TSDB_DATA_TYPE_INT: - comparFn = compareInt32Val; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - comparFn = compareInt64Val; - break; - case TSDB_DATA_TYPE_BOOL: - comparFn = compareInt32Val; - break; - - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - comparFn = compareDoubleVal; - break; - - case TSDB_DATA_TYPE_BINARY: - comparFn = compareStrVal; - break; - - case TSDB_DATA_TYPE_NCHAR: - comparFn = compareWStrVal; - break; - - default: - comparFn = compareInt32Val; - break; - } - - return comparFn; -} +// break; +// } +// case TSDB_DATA_TYPE_BINARY: +// comparFn = compareStrVal; +// break; +// case TSDB_DATA_TYPE_NCHAR: +// comparFn = compareWStrVal; +// break; +// default: +// comparFn = compareInt32Val; +// break; +// } +// +// return comparFn; +//} static bool initForwardBackwardPtr(SSkipList* pSkipList) { uint32_t maxLevel = pSkipList->maxLevel; @@ -298,7 +157,7 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, ui pSkipList->keyInfo = (SSkipListKeyInfo){.type = keyType, .len = keyLen, .dupKey = dupKey, .freeNode = freeNode}; pSkipList->keyFn = fn; - pSkipList->comparFn = getKeyComparator(keyType); + pSkipList->comparFn = getKeyComparFunc(keyType); pSkipList->maxLevel = maxLevel; pSkipList->level = 1; @@ -406,22 +265,18 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { SSkipListNode *px = pSkipList->pHead; SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; - bool identical = false; + int32_t ret = -1; for (int32_t i = pSkipList->level - 1; i >= 0; --i) { SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i); while (p != pSkipList->pTail) { char *key = SL_GET_NODE_KEY(pSkipList, p); // if the forward element is less than the specified key, forward one step - int32_t ret = pSkipList->comparFn(key, newDatakey); + ret = pSkipList->comparFn(key, newDatakey); if (ret < 0) { px = p; p = SL_GET_FORWARD_POINTER(px, i); } else { - if (identical == false) { - identical = (ret == 0); - } - break; } } @@ -430,7 +285,7 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { } // if the skip list does not allowed identical key inserted, the new data will be discarded. - if (pSkipList->keyInfo.dupKey == 0 && identical) { + if (pSkipList->keyInfo.dupKey == 0 && ret == 0) { if (pSkipList->lock) { pthread_rwlock_unlock(pSkipList->lock); } @@ -447,20 +302,13 @@ void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListN for (int32_t i = 0; i < pNode->level; ++i) { SSkipListNode *x = forward[i]; - -// if (x != pSkipList->pTail) { - SL_GET_BACKWARD_POINTER(pNode, i) = x; + SL_GET_BACKWARD_POINTER(pNode, i) = x; - SSkipListNode *next = SL_GET_FORWARD_POINTER(x, i); -// if (next) { - SL_GET_BACKWARD_POINTER(next, i) = pNode; -// } + SSkipListNode *next = SL_GET_FORWARD_POINTER(x, i); + SL_GET_BACKWARD_POINTER(next, i) = pNode; - SL_GET_FORWARD_POINTER(pNode, i) = next; - SL_GET_FORWARD_POINTER(x, i) = pNode; -// } else { -// SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode; -// } + SL_GET_FORWARD_POINTER(pNode, i) = next; + SL_GET_FORWARD_POINTER(x, i) = pNode; } atomic_add_fetch_32(&pSkipList->size, 1); @@ -494,11 +342,9 @@ SSkipListNode* tSkipListDoAppend(SSkipList *pSkipList, SSkipListNode *pNode) { SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType) { int32_t sLevel = pSkipList->level - 1; - int32_t ret = -1; - + // result list SArray* sa = taosArrayInit(1, POINTER_BYTES); - SSkipListNode *pNode = pSkipList->pHead; if (pSkipList->lock) { @@ -509,15 +355,16 @@ SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType) { pSkipList->state.queryCount++; #endif - __compar_fn_t filterComparFn = getKeyFilterComparator(pSkipList, keyType); - + __compar_fn_t filterComparFn = getComparFunc(pSkipList->keyInfo.type, keyType); + int32_t ret = -1; for (int32_t i = sLevel; i >= 0; --i) { - SSkipListNode *pNext = SL_GET_FORWARD_POINTER(pNode, i); - while (pNext != NULL) { - char *key = SL_GET_NODE_KEY(pSkipList, pNext); + SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, i); + while (p != pSkipList->pTail) { + char *key = SL_GET_NODE_KEY(pSkipList, p); + if ((ret = filterComparFn(key, pKey)) < 0) { - pNode = pNext; - pNext = SL_GET_FORWARD_POINTER(pNext, i); + pNode = p; + p = SL_GET_FORWARD_POINTER(p, i); } else { break; } @@ -525,10 +372,6 @@ SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType) { // find the qualified key if (ret == 0) { - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - SSkipListNode* pResult = SL_GET_FORWARD_POINTER(pNode, i); taosArrayPush(sa, &pResult); @@ -559,25 +402,57 @@ SSkipListIterator* tSkipListCreateIter(SSkipList *pSkipList) { return NULL; } - SSkipListIterator* iter = calloc(1, sizeof(SSkipListIterator)); - - iter->pSkipList = pSkipList; - if (pSkipList->lock) { - pthread_rwlock_rdlock(pSkipList->lock); + return doCreateSkipListIterator(pSkipList, TSDB_ORDER_ASC); +} + +SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order) { + if (pSkipList == NULL) { + return NULL; } - iter->cur = NULL; - iter->num = pSkipList->size; + assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); + if (val == NULL) { + return doCreateSkipListIterator(pSkipList, order); + } else { + + SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; + + int32_t ret = -1; + __compar_fn_t filterComparFn = getComparFunc(pSkipList->keyInfo.type, type); + SSkipListNode* pNode = pSkipList->pHead; + + for (int32_t i = pSkipList->level - 1; i >= 0; --i) { + SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, i); + while (p != pSkipList->pTail) { + char *key = SL_GET_NODE_KEY(pSkipList, p); + + if ((ret = filterComparFn(key, val)) < 0) { + pNode = p; + p = SL_GET_FORWARD_POINTER(p, i); + } else { + break; + } + } + + forward[i] = pNode; + } + + SSkipListIterator* iter = doCreateSkipListIterator(pSkipList, order); + + // set the initial position + if (order == TSDB_ORDER_ASC) { + iter->cur = forward[0]; // greater equals than the value + } else { + iter->cur = SL_GET_FORWARD_POINTER(forward[0], 0); + } + + return iter; } - - return iter; } bool tSkipListIterNext(SSkipListIterator *iter) { - if (iter->num == 0 || iter->pSkipList == NULL) { + if (iter->pSkipList == NULL) { return false; } @@ -587,26 +462,35 @@ bool tSkipListIterNext(SSkipListIterator *iter) { pthread_rwlock_rdlock(pSkipList->lock); } - if (iter->cur == NULL) { - iter->cur = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); - } else { - iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); + if (iter->order == TSDB_ORDER_ASC) { // ascending order iterate + if (iter->cur == NULL) { + iter->cur = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); + } else { + iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); + } + } else { // descending order iterate + if (iter->cur == NULL) { + iter->cur = SL_GET_BACKWARD_POINTER(pSkipList->pTail, 0); + } else { + iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0); + } } if (pSkipList->lock) { pthread_rwlock_unlock(pSkipList->lock); } - return iter->cur != pSkipList->pTail; + return (iter->order == TSDB_ORDER_ASC)? (iter->cur != pSkipList->pTail) : (iter->cur != pSkipList->pHead); } SSkipListNode *tSkipListIterGet(SSkipListIterator *iter) { - if (iter == NULL || iter->cur == iter->pSkipList->pTail) { + if (iter == NULL || iter->cur == iter->pSkipList->pTail || iter->cur == iter->pSkipList->pHead) { return NULL; } else { return iter->cur; } } + void* tSkipListDestroyIter(SSkipListIterator* iter) { if (iter == NULL) { return NULL; @@ -622,7 +506,7 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // SSkipListNode *p = pStartNode; // int32_t numOfRes = 0; // -// __compar_fn_t filterComparFn = getKeyFilterComparator(pSkipList, pEndKey->nType); +// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pEndKey->nType); // while (p != NULL) { // int32_t ret = filterComparFn(&p->key, pEndKey); // if (ret > 0) { @@ -663,7 +547,7 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // return 0; // } // -// __compar_fn_t filterComparFn = getKeyFilterComparator(pSkipList, pKey->nType); +// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); // // // backward check if previous nodes are with the same value. // SSkipListNode *pPrev = pNode->pBackward[0]; @@ -679,11 +563,11 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // int32_t ret = -1; // // SSkipListNode *x = &pSkipList->pHead; -// __compar_fn_t filterComparFn = getKeyFilterComparator(pSkipList, pKey->nType); +// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); // // pthread_rwlock_rdlock(&pSkipList->lock); // -// if (cond == TSDB_RELATION_LARGE_EQUAL || cond == TSDB_RELATION_LARGE) { +// if (cond == TSDB_RELATION_GREATER_EQUAL || cond == TSDB_RELATION_GREATER) { // for (int32_t i = sLevel; i >= 0; --i) { // while (x->pForward[i] != NULL && (ret = filterComparFn(&x->pForward[i]->key, pKey)) < 0) { // x = x->pForward[i]; @@ -691,7 +575,7 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // } // // // backward check if previous nodes are with the same value. -// if (cond == TSDB_RELATION_LARGE_EQUAL && ret == 0) { +// if (cond == TSDB_RELATION_GREATER_EQUAL && ret == 0) { // SSkipListNode *pNode = x->pForward[0]; // while ((pNode->pBackward[0] != &pSkipList->pHead) && (filterComparFn(&pNode->pBackward[0]->key, pKey) == 0)) { // pNode = pNode->pBackward[0]; @@ -700,10 +584,10 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // return pNode; // } // -// if (ret > 0 || cond == TSDB_RELATION_LARGE_EQUAL) { +// if (ret > 0 || cond == TSDB_RELATION_GREATER_EQUAL) { // pthread_rwlock_unlock(&pSkipList->lock); // return x->pForward[0]; -// } else { // cond == TSDB_RELATION_LARGE && ret == 0 +// } else { // cond == TSDB_RELATION_GREATER && ret == 0 // SSkipListNode *pn = x->pForward[0]; // while (pn != NULL && filterComparFn(&pn->key, pKey) == 0) { // pn = pn->pForward[0]; @@ -717,19 +601,9 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // return NULL; //} // -// int32_t tSkipListRangeQuery(SSkipList *pSkipList, tSKipListQueryCond *pCond, SSkipListNode ***pRes) { -// pSkipList->state.queryCount++; -// SSkipListNode *pStart = tSkipListParQuery(pSkipList, &pCond->lowerBnd, pCond->lowerBndRelOptr); -// if (pStart == 0) { -// *pRes = NULL; -// return 0; -// } -// -// return tSkipListEndParQuery(pSkipList, pStart, &pCond->upperBnd, pCond->upperBndRelOptr, pRes); -//} // // static bool removeSupport(SSkipList *pSkipList, SSkipListNode **forward, SSkipListKey *pKey) { -// __compar_fn_t filterComparFn = getKeyFilterComparator(pSkipList, pKey->nType); +// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); // // if (filterComparFn(&forward[0]->pForward[0]->key, pKey) == 0) { // SSkipListNode *p = forward[0]->pForward[0]; @@ -760,7 +634,7 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // // bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey *pKey) { // SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; -// __compar_fn_t filterComparFn = getKeyFilterComparator(pSkipList, pKey->nType); +// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); // // pthread_rwlock_rdlock(&pSkipList->lock); // @@ -818,3 +692,12 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { p = SL_GET_FORWARD_POINTER(p, nlevel - 1); } } + +SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) { + SSkipListIterator* iter = calloc(1, sizeof(SSkipListIterator)); + + iter->pSkipList = pSkipList; + iter->order = order; + + return iter; +} diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index cadb2e706c..ba06d73352 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -164,9 +164,10 @@ char* strtolower(char *dst, const char *src) { return dst; } -char *paGetToken(char *string, char **token, int32_t *tokenLen) { +char *paGetToken(char *string, size_t maxLen, char **token, int32_t *tokenLen) { char quote = 0; - + char* start = string; + while (*string != 0) { if (*string == ' ' || *string == '\t') { ++string; @@ -182,7 +183,7 @@ char *paGetToken(char *string, char **token, int32_t *tokenLen) { *token = string; - while (*string != 0) { + while (*string != 0 && *string != '\n' && (string - start < maxLen)) { if (*string == '@' && quote) { //*string = 0; ++string; diff --git a/src/util/tests/skiplistTest.cpp b/src/util/tests/skiplistTest.cpp index 9e690c3924..8bfa3a43c0 100644 --- a/src/util/tests/skiplistTest.cpp +++ b/src/util/tests/skiplistTest.cpp @@ -104,6 +104,7 @@ void randKeyTest() { tSkipListDestroy(pSkipList); } + void stringKeySkiplistTest() { const int32_t max_key_size = 12; From e4d2b5bb5e0d82c0843ea4519e3da782396592d8 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 10 Apr 2020 15:09:37 +0800 Subject: [PATCH 34/78] turn on the arbitrator code --- src/inc/tsync.h | 1 + src/vnode/main/src/vnodeMain.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/inc/tsync.h b/src/inc/tsync.h index ea20ceec79..cdc92b2366 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -99,6 +99,7 @@ extern int tsMaxSyncNum; extern int tsSyncTcpThreads; extern int tsMaxWatchFiles; extern short tsSyncPort; +extern int tsSyncTimer; extern int tsMaxFwdInfo; extern int sDebugFlag; diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 8d36f59da0..7ee468fa54 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -375,7 +375,7 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT; if (strcmp(option[0], "arbitratorIp") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; if (arbitratorIp == -1) return TSDB_CODE_INVALID_FILE_FORMAT; - pVnode->syncCfg.arbitratorIp = 0; + pVnode->syncCfg.arbitratorIp = arbitratorIp; int32_t quorum = -1; num = fscanf(fp, "%s %d", option[0], &quorum); From 0c2e506772d234f39c1593a5dc2092c65d6d4040 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 10 Apr 2020 17:35:29 +0800 Subject: [PATCH 35/78] add test --- tests/CMakeLists.txt | 1 + tests/test/c/CMakeLists.txt | 11 ++ tests/test/c/benchmarkPerTable.c | 319 +++++++++++++++++++++++++++++++ 3 files changed, 331 insertions(+) create mode 100644 tests/test/c/CMakeLists.txt create mode 100644 tests/test/c/benchmarkPerTable.c diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 872201f178..a6070a2fc1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -11,3 +11,4 @@ SET(CMAKE_VERBOSE_MAKEFILE ON) ADD_SUBDIRECTORY(examples/c) ADD_SUBDIRECTORY(tsim) +ADD_SUBDIRECTORY(test/c) diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt new file mode 100644 index 0000000000..532f304b99 --- /dev/null +++ b/tests/test/c/CMakeLists.txt @@ -0,0 +1,11 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) +INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) + +IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) + add_executable(benchmarkPerTable benchmarkPerTable.c) + target_link_libraries(benchmarkPerTable taos_static pthread) +ENDIF() diff --git a/tests/test/c/benchmarkPerTable.c b/tests/test/c/benchmarkPerTable.c new file mode 100644 index 0000000000..ee4e0fecbd --- /dev/null +++ b/tests/test/c/benchmarkPerTable.c @@ -0,0 +1,319 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taos.h" +#include "tlog.h" +#include "ttimer.h" +#include "tutil.h" + +#define MAX_RANDOM_POINTS 20000 +#define GREEN "\033[1;32m" +#define NC "\033[0m" + +typedef struct { + int64_t rowsPerTable; + int64_t pointsPerTable; + int64_t tableBeginIndex; + int64_t tableEndIndex; + int threadIndex; + char dbName[32]; + char stableName[64]; + pthread_t thread; +} SInfo; + +void *syncTest(void *param); +void generateRandomPoints(); +void shellParseArgument(int argc, char *argv[]); +void createDbAndTable(); +void insertData(); + +int32_t randomData[MAX_RANDOM_POINTS]; +int64_t rowsPerTable = 10000; +int64_t pointsPerTable = 1; +int64_t numOfThreads = 1; +int64_t numOfTablesPerThread = 1; +char dbName[32] = "db"; +char stableName[64] = "st"; +int32_t cache = 16384; +int32_t tables = 1000; + +int main(int argc, char *argv[]) { + shellParseArgument(argc, argv); + generateRandomPoints(); + taos_init(); + createDbAndTable(); + insertData(); +} + +void createDbAndTable() { + dPrint("start to create table"); + + TAOS * con; + struct timeval systemTime; + int64_t st, et; + char qstr[64000]; + + con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0); + if (con == NULL) { + dError("failed to connect to DB, reason:%s", taos_errstr(con)); + exit(1); + } + + sprintf(qstr, "create database if not exists %s cache %d tables %d", dbName, cache, tables); + if (taos_query(con, qstr)) { + dError("failed to create database:%s, code:%d reason:%s", dbName, taos_errno(con), taos_errstr(con)); + exit(0); + } + + sprintf(qstr, "use %s", dbName); + if (taos_query(con, qstr)) { + dError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con)); + exit(0); + } + + gettimeofday(&systemTime, NULL); + st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + int64_t totalTables = numOfTablesPerThread * numOfThreads; + + if (strcmp(stableName, "no") != 0) { + int len = sprintf(qstr, "create table if not exists %s(ts timestamp", stableName); + for (int64_t f = 0; f < pointsPerTable; ++f) { + len += sprintf(qstr + len, ", f%ld double", f); + } + sprintf(qstr + len, ") tags(t int)"); + + if (taos_query(con, qstr)) { + dError("failed to create stable, code:%d reason:%s", taos_errno(con), taos_errstr(con)); + exit(0); + } + + for (int64_t t = 0; t < totalTables; ++t) { + sprintf(qstr, "create table if not exists %s%ld using %s tags(%ld)", stableName, t, stableName, t); + if (taos_query(con, qstr)) { + dError("failed to create table %s%d, reason:%s", stableName, t, taos_errstr(con)); + exit(0); + } + } + } else { + for (int64_t t = 0; t < totalTables; ++t) { + int len = sprintf(qstr, "create table if not exists %s%ld(ts timestamp", stableName, t); + for (int64_t f = 0; f < pointsPerTable; ++f) { + len += sprintf(qstr + len, ", f%ld double", f); + } + sprintf(qstr + len, ")"); + + if (taos_query(con, qstr)) { + dError("failed to create table %s%ld, reason:%s", stableName, t, taos_errstr(con)); + exit(0); + } + } + } + + gettimeofday(&systemTime, NULL); + et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + dPrint("%.1f seconds to create %ld tables", (et - st) / 1000.0 / 1000.0, totalTables); +} + +void insertData() { + struct timeval systemTime; + int64_t st, et; + + gettimeofday(&systemTime, NULL); + st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + + dPrint("%d threads are spawned to insert data", numOfThreads); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + SInfo *pInfo = (SInfo *)malloc(sizeof(SInfo) * numOfThreads); + + // Start threads to write + for (int i = 0; i < numOfThreads; ++i) { + pInfo[i].rowsPerTable = rowsPerTable; + pInfo[i].pointsPerTable = pointsPerTable; + pInfo[i].tableBeginIndex = i * numOfTablesPerThread; + pInfo[i].tableEndIndex = (i + 1) * numOfTablesPerThread; + pInfo[i].threadIndex = i; + strcpy(pInfo[i].dbName, dbName); + strcpy(pInfo[i].stableName, stableName); + pthread_create(&(pInfo[i].thread), &thattr, syncTest, (void *)(pInfo + i)); + } + + taosMsleep(300); + for (int i = 0; i < numOfThreads; i++) { + pthread_join(pInfo[i].thread, NULL); + } + + gettimeofday(&systemTime, NULL); + et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + double seconds = (et - st) / 1000.0 / 1000.0; + + int64_t totalTables = numOfTablesPerThread * numOfThreads; + int64_t totalRows = totalTables * rowsPerTable; + int64_t totalPoints = totalTables * rowsPerTable * pointsPerTable; + double speedOfRows = totalRows / seconds; + double speedOfPoints = totalPoints / seconds; + + dPrint( + "%sall threads:%ld finished, use %.1lf seconds, tables:%.ld rows:%ld points:%ld, speed RowsPerSecond:%.1lf " + "PointsPerSecond:%.1lf%s", + GREEN, numOfThreads, seconds, totalTables, totalRows, totalPoints, speedOfRows, speedOfPoints, NC); + + dPrint("threads exit"); + + pthread_attr_destroy(&thattr); + free(pInfo); +} + +void *syncTest(void *param) { + TAOS * con; + SInfo * pInfo = (SInfo *)param; + struct timeval systemTime; + int64_t st, et; + char qstr[65000]; + int maxBytes = 60000; + + dPrint("thread:%d, start to run", pInfo->threadIndex); + + con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0); + if (con == NULL) { + dError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); + exit(1); + } + + sprintf(qstr, "use %s", pInfo->dbName); + taos_query(con, qstr); + + gettimeofday(&systemTime, NULL); + st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + + int64_t start = 1430000000000; + int64_t interval = 1000; // 1000 ms + + char *sql = qstr; + char inserStr[] = "insert into"; + int len = sprintf(sql, "%s", inserStr); + + for (int64_t table = pInfo->tableBeginIndex; table < pInfo->tableEndIndex; ++table) { + len += sprintf(sql + len, " %s%ld values", pInfo->stableName, table); + for (int64_t row = 0; row < pInfo->rowsPerTable; row++) { + len += sprintf(sql + len, "(%ld", start + row * interval); + for (int64_t point = 0; point < pInfo->pointsPerTable; ++point) { + len += sprintf(sql + len, ",%d", randomData[(123 * table + 456 * row + 789 * point) % MAX_RANDOM_POINTS]); + // len += sprintf(sql + len, ",%ld", row); + } + len += sprintf(sql + len, ")"); + if (len > maxBytes) { + if (taos_query(con, qstr)) { + dError("thread:%d, failed to insert table:%s%ld row:%ld, reason:%s", pInfo->threadIndex, pInfo->stableName, + table, row, taos_errstr(con)); + } + + // "insert into" + len = sprintf(sql, "%s", inserStr); + + // "insert into st1 values" + if (row != pInfo->rowsPerTable - 1) { + len += sprintf(sql + len, " %s%ld values", pInfo->stableName, table); + } + } + } + } + + if (len != strlen(inserStr)) { + taos_query(con, qstr); + } + + gettimeofday(&systemTime, NULL); + et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + int64_t totalTables = pInfo->tableEndIndex - pInfo->tableBeginIndex; + int64_t totalRows = totalTables * pInfo->rowsPerTable; + int64_t totalPoints = totalRows * pInfo->pointsPerTable; + dPrint("thread:%d, insert finished, use %.2f seconds, tables:%ld rows:%ld points:%ld", pInfo->threadIndex, + (et - st) / 1000.0 / 1000.0, totalTables, totalRows, totalPoints); + + return NULL; +} + +void generateRandomPoints() { + for (int r = 0; r < MAX_RANDOM_POINTS; ++r) { + randomData[r] = rand() % 1000; + } +} + +void printHelp() { + char indent[10] = " "; + printf("Used to test the performance of TDengine, the insert method is table-by-table\n"); + + printf("%s%s\n", indent, "-d"); + printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", stableName, ", if 'no' then create normal table"); + printf("%s%s\n", indent, "-c"); + printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%ld\n", indent, indent, "Number of records to write to each table, default is ", rowsPerTable); + printf("%s%s\n", indent, "-p"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of columns per table, default is ", pointsPerTable); + printf("%s%s\n", indent, "-t"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of threads to be used, default is ", numOfThreads); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of tables per thread, default is ", numOfTablesPerThread); + printf("%s%s\n", indent, "-tables"); + printf("%s%s%s%d\n", indent, indent, "Database parameters tables, default is ", tables); + printf("%s%s\n", indent, "-cache"); + printf("%s%s%s%d\n", indent, indent, "Database parameters cache, default is ", cache); + + exit(EXIT_SUCCESS); +} + +void shellParseArgument(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-d") == 0) { + strcpy(dbName, argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + strcpy(stableName, argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + rowsPerTable = atoi(argv[++i]); + } else if (strcmp(argv[i], "-p") == 0) { + pointsPerTable = atoi(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + numOfTablesPerThread = atoi(argv[++i]); + } else if (strcmp(argv[i], "-tables") == 0) { + tables = atoi(argv[++i]); + } else if (strcmp(argv[i], "-cache") == 0) { + cache = atoi(argv[++i]); + } else { + } + } + + dPrint("%srowsPerTable:%" PRId64 "%s", GREEN, rowsPerTable, NC); + dPrint("%spointsPerTable:%" PRId64 "%s", GREEN, pointsPerTable, NC); + dPrint("%snumOfThreads:%" PRId64 "%s", GREEN, numOfThreads, NC); + dPrint("%snumOfTablesPerThread:%" PRId64 "%s", GREEN, numOfTablesPerThread, NC); + dPrint("%scache:%" PRId64 "%s", GREEN, cache, NC); + dPrint("%stables:%" PRId64 "%s", GREEN, tables, NC); + dPrint("%sdbName:%s%s", GREEN, dbName, NC); + dPrint("%stableName:%s%s", GREEN, stableName, NC); + dPrint("%sstart to run%s", GREEN, NC); +} From 7b900a5dd120b20380a68bfa13b84b18e3b7e2a7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 10 Apr 2020 22:50:31 +0800 Subject: [PATCH 36/78] check null pointer in tscSql.c. --- src/client/src/tscSql.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 199b4150e8..f2a3cdcab0 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -329,7 +329,10 @@ int taos_num_fields(TAOS_RES *res) { } SFieldInfo *pFieldsInfo = &pQueryInfo->fieldsInfo; - return (pFieldsInfo->numOfOutputCols - pFieldsInfo->numOfHiddenCols); + if (pFieldsInfo) + return (pFieldsInfo->numOfOutputCols - pFieldsInfo->numOfHiddenCols); + else + return 0; } int taos_field_count(TAOS *taos) { @@ -351,7 +354,11 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { if (pSql == NULL || pSql->signature != pSql) return 0; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - return pQueryInfo->fieldsInfo.pFields; + + if (pQueryInfo) + return pQueryInfo->fieldsInfo.pFields; + else + return NULL; } int taos_retrieve(TAOS_RES *res) { @@ -401,6 +408,9 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) { } SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + if (pQueryInfo == NULL) + return 0; + for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i); } From 235b0da12e7075a9ab85f5f78e3745fe9bff718a Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 10 Apr 2020 23:08:50 +0800 Subject: [PATCH 37/78] [td-98] sleep a little bit longer --- src/dnode/src/dnodeWrite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnode/src/dnodeWrite.c b/src/dnode/src/dnodeWrite.c index 2e0ec5f95b..3c598ca360 100644 --- a/src/dnode/src/dnodeWrite.c +++ b/src/dnode/src/dnodeWrite.c @@ -218,7 +218,7 @@ static void dnodeHandleIdleWorker(SWriteWorker *pWorker) { int32_t num = taosGetQueueNumber(pWorker->qset); if (num > 0) { - usleep(100); + usleep(1000); sched_yield(); } else { taosCloseQset(pWorker->qset); From 3f9d5ccaa4cc157a18d165adb8e38ee4ed2cd38a Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 10 Apr 2020 23:15:33 +0800 Subject: [PATCH 38/78] [td-98] fix bug for query table, tag query cond serialization, --- src/client/inc/tscUtil.h | 7 +- src/client/inc/tsclient.h | 8 +- src/client/src/tscAsync.c | 4 +- src/client/src/tscFunctionImpl.c | 56 +- src/client/src/tscLocal.c | 6 +- src/client/src/tscSQLParser.c | 231 +++-- src/client/src/tscSecondaryMerge.c | 8 +- src/client/src/tscServer.c | 151 ++- src/client/src/tscSql.c | 6 +- src/client/src/tscSubquery.c | 29 +- src/client/src/tscUtil.c | 131 ++- src/inc/taosdef.h | 8 +- src/inc/taosmsg.h | 14 +- src/query/inc/qast.h | 6 +- src/query/inc/queryExecutor.h | 6 +- src/query/inc/sql.y | 6 +- src/query/inc/tsqlfunction.h | 2 +- src/query/src/qast.c | 227 ++-- src/query/src/qextbuffer.c | 4 +- src/query/src/qinterpolation.c | 4 +- src/query/src/qpercentile.c | 2 +- src/query/src/qsyntaxtreefunction.c | 32 +- src/query/src/qtsbuf.c | 54 +- src/query/src/queryExecutor.c | 458 ++++----- src/query/src/sql.c | 6 +- src/util/inc/tcompare.h | 43 + src/util/src/tcompare.c | 192 ++++ src/vnode/tsdb/inc/tsdb.h | 4 +- src/vnode/tsdb/src/tsdbMain.c | 4 +- src/vnode/tsdb/src/tsdbMeta.c | 13 +- src/vnode/tsdb/src/tsdbRead.c | 1483 +++++++++++++++------------ 31 files changed, 1862 insertions(+), 1343 deletions(-) create mode 100644 src/util/inc/tcompare.h create mode 100644 src/util/src/tcompare.c diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 0b6ae06416..6e8559538d 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -24,6 +24,7 @@ extern "C" { * @date 2018/09/30 */ #include "os.h" +#include "tbuffer.h" #include "qextbuffer.h" #include "taosdef.h" #include "tscSecondaryMerge.h" @@ -176,8 +177,8 @@ void tscIncStreamExecutionCount(void* pStream); bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId); // get starter position of metric query condition (query on tags) in SSqlCmd.payload -SCond* tsGetSTableQueryCondPos(STagCond* pCond, uint64_t tableIndex); -void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, const char* str); +SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid); +void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBuffer* pBuf); void tscTagCondCopy(STagCond* dest, const STagCond* src); void tscTagCondRelease(STagCond* pCond); @@ -199,7 +200,7 @@ int32_t tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex, SQuer STableMetaInfo* tscGetMeterMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index); void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); -STableMetaInfo* tscAddMeterMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, SSuperTableMeta* pMetricMeta, +STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, SArray* vgroupList, int16_t numOfTags, int16_t* tags); STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo); int32_t tscAddSubqueryInfo(SSqlCmd *pCmd); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 80c3e77eff..5c442a155b 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -43,7 +43,7 @@ struct SSqlInfo; typedef struct SSqlGroupbyExpr { int16_t tableIndex; int16_t numOfGroupCols; - SColIndexEx columnInfo[TSDB_MAX_TAGS]; // group by columns information + SColIndex columnInfo[TSDB_MAX_TAGS]; // group by columns information int16_t orderIndex; // order by column index int16_t orderType; // order by type: asc/desc } SSqlGroupbyExpr; @@ -86,7 +86,7 @@ typedef struct STableMetaInfo { /* the structure for sql function in select clause */ typedef struct SSqlExpr { char aliasName[TSDB_COL_NAME_LEN]; // as aliasName - SColIndexEx colInfo; + SColIndex colInfo; int64_t uid; // refactor use the pointer int16_t functionId; // function id in aAgg array int16_t resType; // return value type @@ -141,6 +141,7 @@ struct SLocalReducer; typedef struct SCond { uint64_t uid; + int32_t len; // length of tag query condition data char * cond; } SCond; @@ -167,8 +168,7 @@ typedef struct STagCond { SJoinInfo joinInfo; // for different table, the query condition must be seperated - SCond cond[TSDB_MAX_JOIN_TABLE_NUM]; - int16_t numOfTagCond; + SArray* pCond; } STagCond; typedef struct SParamInfo { diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 36cd42332b..0c24398fea 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -464,13 +464,15 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; } else { // normal async query continues if (pCmd->isParseFinish) { - tscTrace("%p resend data to vnode in metermeta callback since sql has been parsed completed", pSql); + tscTrace("%p re-send data to vnode in table Meta callback since sql parsed completed", pSql); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); code = tscGetTableMeta(pSql, pTableMetaInfo); assert(code == TSDB_CODE_SUCCESS); if (pTableMetaInfo->pTableMeta) { + // todo update the submit message according to the new table meta + // 1. table uid, 2. ip address code = tscSendMsgToServer(pSql); if (code == TSDB_CODE_SUCCESS) return; } diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index c7be1b84ce..0fa9e01c2e 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -691,7 +691,7 @@ static int32_t data_req_load_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, // todo: if column in current data block are null, opt for this case static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { - if (pCtx->order == TSQL_SO_DESC) { + if (pCtx->order == TSDB_ORDER_DESC) { return BLK_DATA_NO_NEEDED; } @@ -704,7 +704,7 @@ static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, } static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { - if (pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSDB_ORDER_ASC) { return BLK_DATA_NO_NEEDED; } @@ -716,7 +716,7 @@ static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, } static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { - if (pCtx->order == TSQL_SO_DESC) { + if (pCtx->order == TSDB_ORDER_DESC) { return BLK_DATA_NO_NEEDED; } @@ -732,7 +732,7 @@ static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY } static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { - if (pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSDB_ORDER_ASC) { return BLK_DATA_NO_NEEDED; } @@ -1483,7 +1483,7 @@ static bool first_last_function_setup(SQLFunctionCtx *pCtx) { // todo opt for null block static void first_function(SQLFunctionCtx *pCtx) { - if (pCtx->order == TSQL_SO_DESC) { + if (pCtx->order == TSDB_ORDER_DESC) { return; } @@ -1513,7 +1513,7 @@ static void first_function(SQLFunctionCtx *pCtx) { } static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { - if (pCtx->order == TSQL_SO_DESC) { + if (pCtx->order == TSDB_ORDER_DESC) { return; } @@ -1561,7 +1561,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { * 1. data block that are not loaded * 2. scan data files in desc order */ - if (pCtx->order == TSQL_SO_DESC) { + if (pCtx->order == TSDB_ORDER_DESC) { return; } @@ -1596,7 +1596,7 @@ static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - if (pCtx->order == TSQL_SO_DESC) { + if (pCtx->order == TSDB_ORDER_DESC) { return; } @@ -1654,7 +1654,7 @@ static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) { * least one data in this block that is not null.(TODO opt for this case) */ static void last_function(SQLFunctionCtx *pCtx) { - if (pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSDB_ORDER_ASC) { return; } @@ -1683,7 +1683,7 @@ static void last_function(SQLFunctionCtx *pCtx) { } static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { - if (pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSDB_ORDER_ASC) { return; } @@ -1730,7 +1730,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { * 1. for scan data in asc order, no need to check data * 2. for data blocks that are not loaded, no need to check data */ - if (pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSDB_ORDER_ASC) { return; } @@ -1768,7 +1768,7 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { * 1. for scan data in asc order, no need to check data * 2. for data blocks that are not loaded, no need to check data */ - if (pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSDB_ORDER_ASC) { return; } @@ -2420,10 +2420,10 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { // user specify the order of output by sort the result according to timestamp if (pCtx->param[1].i64Key == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - __compar_fn_t comparator = (pCtx->param[2].i64Key == TSQL_SO_ASC) ? resAscComparFn : resDescComparFn; + __compar_fn_t comparator = (pCtx->param[2].i64Key == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); } else if (pCtx->param[1].i64Key > PRIMARYKEY_TIMESTAMP_COL_INDEX) { - __compar_fn_t comparator = (pCtx->param[2].i64Key == TSQL_SO_ASC) ? resDataAscComparFn : resDataDescComparFn; + __compar_fn_t comparator = (pCtx->param[2].i64Key == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); } @@ -2449,7 +2449,7 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) { int32_t orderIdx = 0; // tOrderDesc object - tOrderDescriptor *pDesc = tOrderDesCreate(&orderIdx, NUMOFCOLS, pModel, TSQL_SO_DESC); + tOrderDescriptor *pDesc = tOrderDesCreate(&orderIdx, NUMOFCOLS, pModel, TSDB_ORDER_DESC); ((SPercentileInfo *)(pResInfo->interResultBuf))->pMemBucket = tMemBucketCreate(1024, MAX_AVAILABLE_BUFFER_SIZE, pCtx->inputBytes, pCtx->inputType, pDesc); @@ -2916,7 +2916,7 @@ static void col_project_function(SQLFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); char *pData = GET_INPUT_CHAR(pCtx); - if (pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSDB_ORDER_ASC) { memcpy(pCtx->aOutputBuf, pData, (size_t)pCtx->size * pCtx->inputBytes); } else { for(int32_t i = 0; i < pCtx->size; ++i) { @@ -3011,7 +3011,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSQL_SO_ASC) ? 0 : pCtx->size - 1; + int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; TSKEY * pTimestamp = pCtx->ptsOutputBuf; @@ -3028,7 +3028,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; - } else if ((i == 0 && pCtx->order == TSQL_SO_ASC) || (i == pCtx->size - 1 && pCtx->order == TSQL_SO_DESC)) { + } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].i64Key; *pTimestamp = pCtx->ptsList[i]; @@ -3060,7 +3060,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; - } else if ((i == 0 && pCtx->order == TSQL_SO_ASC) || (i == pCtx->size - 1 && pCtx->order == TSQL_SO_DESC)) { + } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].i64Key; *pTimestamp = pCtx->ptsList[i]; @@ -3092,7 +3092,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].dKey = pData[i]; pCtx->param[1].nType = pCtx->inputType; - } else if ((i == 0 && pCtx->order == TSQL_SO_ASC) || (i == pCtx->size - 1 && pCtx->order == TSQL_SO_DESC)) { + } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].dKey; *pTimestamp = pCtx->ptsList[i]; pOutput += 1; @@ -3122,7 +3122,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].dKey = pData[i]; pCtx->param[1].nType = pCtx->inputType; - } else if ((i == 0 && pCtx->order == TSQL_SO_ASC) || (i == pCtx->size - 1 && pCtx->order == TSQL_SO_DESC)) { + } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].dKey; *pTimestamp = pCtx->ptsList[i]; @@ -3155,7 +3155,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; - } else if ((i == 0 && pCtx->order == TSQL_SO_ASC) || (i == pCtx->size - 1 && pCtx->order == TSQL_SO_DESC)) { + } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].i64Key; *pTimestamp = pCtx->ptsList[i]; pOutput += 1; @@ -3186,7 +3186,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; - } else if ((i == 0 && pCtx->order == TSQL_SO_ASC) || (i == pCtx->size - 1 && pCtx->order == TSQL_SO_DESC)) { + } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].i64Key; *pTimestamp = pCtx->ptsList[i]; @@ -3298,17 +3298,17 @@ char *arithmetic_callback_function(void *param, char *name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *)param; SSqlFunctionExpr *pExpr = pSupport->pExpr; - int32_t colIndexInBuf = -1; + int32_t colIndex = -1; for (int32_t i = 0; i < pExpr->binExprInfo.numOfCols; ++i) { if (colId == pExpr->binExprInfo.pReqColumns[i].colId) { - colIndexInBuf = pExpr->binExprInfo.pReqColumns[i].colIdxInBuf; + colIndex = pExpr->binExprInfo.pReqColumns[i].colIndex; break; } } - assert(colIndexInBuf >= 0 && colId >= 0); - return pSupport->data[colIndexInBuf] + pSupport->offset * pSupport->elemSize[colIndexInBuf]; + assert(colIndex >= 0 && colId >= 0); + return pSupport->data[colIndex] + pSupport->offset * pSupport->elemSize[colIndex]; } static void arithmetic_function(SQLFunctionCtx *pCtx) { @@ -4327,7 +4327,7 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { const char *input = GET_INPUT_CHAR(pCtx); // primary ts must be existed, so no need to check its existance - if (pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSDB_ORDER_ASC) { tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, input, pCtx->size * TSDB_KEYSIZE); } else { for (int32_t i = pCtx->size - 1; i >= 0; --i) { diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 1c738ecb3d..41d5febab5 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -240,7 +240,7 @@ static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, pCmd->numOfCols = numOfCols; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - pQueryInfo->order.order = TSQL_SO_ASC; + pQueryInfo->order.order = TSDB_ORDER_ASC; tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_BINARY, "Field", TSDB_COL_NAME_LEN); rowLen += TSDB_COL_NAME_LEN; @@ -322,7 +322,7 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { STableIdInfo *pSidExt = tscGetMeterSidInfo(pSidList, j); for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutputCols; ++k) { - SColIndexEx *pColIndex = &tscSqlExprGet(pQueryInfo, k)->colInfo; + SColIndex *pColIndex = &tscSqlExprGet(pQueryInfo, k)->colInfo; int16_t offsetId = pColIndex->colIdx; assert((pColIndex->flag & TSDB_COL_TAG) != 0); @@ -460,7 +460,7 @@ void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnNa pCmd->numOfCols = 1; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - pQueryInfo->order.order = TSQL_SO_ASC; + pQueryInfo->order.order = TSDB_ORDER_ASC; tscClearFieldInfo(&pQueryInfo->fieldsInfo); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4462af7f95..630e439397 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -45,7 +45,7 @@ typedef struct SColumnList { SColumnIndex ids[TSDB_MAX_COLUMNS]; } SColumnList; -static SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t colIdx, int32_t tableIndex); +static SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t colIndex, int32_t tableIndex); static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); static char* getAccountId(SSqlObj* pSql); @@ -61,7 +61,7 @@ static int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pD static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength); static void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName); -static int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprItem* pItem, bool finalResult); +static int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult); static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, int8_t type, char* fieldName, SSqlExpr* pSqlExpr); static int32_t changeFunctionID(int32_t optr, int16_t* functionId); @@ -116,9 +116,7 @@ static int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSql static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index); - -static int32_t convertSyntaxTreeToExprTree(tExprNode **pExpr, tSQLExpr* pAst, int32_t* num, - SColIndexEx** pColIndex, SSqlExprInfo* pExprInfo); +static int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SSqlExprInfo* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols); /* * Used during parsing query sql. Since the query sql usually small in length, error position @@ -136,7 +134,7 @@ static int32_t tscQueryOnlyMetricTags(SQueryInfo* pQueryInfo, bool* queryOnMetri SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId != TSDB_FUNC_TAGPRJ && - !(pExpr->functionId == TSDB_FUNC_COUNT && pExpr->colInfo.colIdx == TSDB_TBNAME_COLUMN_INDEX)) { + !(pExpr->functionId == TSDB_FUNC_COUNT && pExpr->colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX)) { *queryOnMetricTags = false; break; } @@ -1209,28 +1207,29 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel SSqlBinaryExprInfo* pBinExprInfo = &pFuncExpr->binExprInfo; tExprNode* pNode = NULL; - SColIndexEx* pColIndex = NULL; +// SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - int32_t ret = convertSyntaxTreeToExprTree(&pNode, pItem->pNode, &pBinExprInfo->numOfCols, &pColIndex, &pQueryInfo->exprsInfo); + int32_t ret = exprTreeFromSqlExpr(&pNode, pItem->pNode, &pQueryInfo->exprsInfo, pQueryInfo, NULL); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(pQueryInfo->msg, "invalid expression in select clause"); } pBinExprInfo->pBinExpr = pNode; - pBinExprInfo->pReqColumns = pColIndex; + assert(0); +// pBinExprInfo->pReqColumns = pColIndex; for(int32_t k = 0; k < pBinExprInfo->numOfCols; ++k) { - SColIndexEx* pCol = &pBinExprInfo->pReqColumns[k]; + SColIndex* pCol = &pBinExprInfo->pReqColumns[k]; for(int32_t f = 0; f < pQueryInfo->exprsInfo.numOfExprs; ++f) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, f); if (strcmp(pExpr->aliasName, pCol->name) == 0) { - pCol->colIdxInBuf = f; + pCol->colIndex = f; break; } } - assert(pCol->colIdxInBuf >= 0 && pCol->colIdxInBuf < pQueryInfo->exprsInfo.numOfExprs); + assert(pCol->colIndex >= 0 && pCol->colIndex < pQueryInfo->exprsInfo.numOfExprs); tfree(pNode); } } @@ -1288,23 +1287,23 @@ int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnLi return TSDB_CODE_SUCCESS; } -SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t colIdx, int32_t tableIndex) { +SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t colIndex, int32_t tableIndex) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, colIdx); + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, colIndex); - int16_t functionId = (int16_t)((colIdx >= numOfCols) ? TSDB_FUNC_TAGPRJ : TSDB_FUNC_PRJ); + int16_t functionId = (int16_t)((colIndex >= numOfCols) ? TSDB_FUNC_TAGPRJ : TSDB_FUNC_PRJ); if (functionId == TSDB_FUNC_TAGPRJ) { -// addRequiredTagColumn(pQueryInfo, colIdx - numOfCols, tableIndex); +// addRequiredTagColumn(pQueryInfo, colIndex - numOfCols, tableIndex); pQueryInfo->type = TSDB_QUERY_TYPE_STABLE_QUERY; } else { pQueryInfo->type = TSDB_QUERY_TYPE_PROJECTION_QUERY; } - SColumnIndex index = {tableIndex, colIdx}; + SColumnIndex index = {tableIndex, colIndex}; SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputIndex, functionId, &index, pSchema->type, pSchema->bytes, pSchema->bytes); @@ -1503,7 +1502,7 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema, return TSDB_CODE_SUCCESS; } -int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprItem* pItem, bool finalResult) { +int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult) { STableMetaInfo* pTableMetaInfo = NULL; int32_t optr = pItem->pNode->nSQLOptr; @@ -1547,7 +1546,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, colIdx, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } else { // count the number of meters created according to the metric if (getColumnIndexByName(pToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -1562,13 +1561,13 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt } int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, colIdx, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } } else { // count(*) is equalled to count(primary_timestamp_key) index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, colIdx, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); @@ -1646,7 +1645,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt // set the first column ts for diff query if (optr == TK_DIFF) { - colIdx += 1; + colIndex += 1; SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); @@ -1660,7 +1659,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt return invalidSqlErrMsg(pQueryInfo->msg, msg6); } - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, colIdx, functionID, &index, resultType, resultSize, resultSize); + SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, resultType, resultSize, resultSize); if (optr == TK_LEASTSQUARES) { /* set the leastsquares parameters */ @@ -1737,7 +1736,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) { index.columnIndex = j; - if (setExprInfoForFunctions(pQueryInfo, pSchema, functionID, pItem->aliasName, colIdx++, &index) != 0) { + if (setExprInfoForFunctions(pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex++, &index) != 0) { return TSDB_CODE_INVALID_SQL; } } @@ -1755,7 +1754,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt return invalidSqlErrMsg(pQueryInfo->msg, msg6); } - if (setExprInfoForFunctions(pQueryInfo, pSchema, functionID, pItem->aliasName, colIdx + i, &index) != 0) { + if (setExprInfoForFunctions(pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex + i, &index) != 0) { return TSDB_CODE_INVALID_SQL; } } @@ -1771,7 +1770,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { SColumnIndex index = {.tableIndex = j, .columnIndex = i}; - if (setExprInfoForFunctions(pQueryInfo, pSchema, functionID, pItem->aliasName, colIdx + i + j, &index) != + if (setExprInfoForFunctions(pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex + i + j, &index) != 0) { return TSDB_CODE_INVALID_SQL; } @@ -1852,7 +1851,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt return TSDB_CODE_INVALID_SQL; } - pExpr = tscSqlExprInsert(pQueryInfo, colIdx, functionId, &index, resultType, resultSize, resultSize); + pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionId, &index, resultType, resultSize, resultSize); addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0); } else { tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT); @@ -1876,9 +1875,9 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt insertResultField(pQueryInfo, TS_COLUMN_INDEX, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS].aName, pExpr); - colIdx += 1; // the first column is ts + colIndex += 1; // the first column is ts - pExpr = tscSqlExprInsert(pQueryInfo, colIdx, functionId, &index, resultType, resultSize, resultSize); + pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionId, &index, resultType, resultSize, resultSize); addExprParams(pExpr, val, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0); } @@ -1887,7 +1886,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt SColumnList ids = getColumnList(1, 0, index.columnIndex); if (finalResult) { - insertResultField(pQueryInfo, colIdx, &ids, resultSize, resultType, pExpr->aliasName, pExpr); + insertResultField(pQueryInfo, colIndex, &ids, resultSize, resultType, pExpr->aliasName, pExpr); } else { for (int32_t i = 0; i < ids.num; ++i) { tscColumnBaseInfoInsert(pQueryInfo, &(ids.ids[i])); @@ -2275,7 +2274,7 @@ int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k); int16_t functionId = aAggs[pExpr->functionId].stableFuncId; - int32_t colIndex = pExpr->colInfo.colIdx; + int32_t colIndex = pExpr->colInfo.colIndex; SSchema* pSrcSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, colIndex); if ((functionId >= TSDB_FUNC_SUM && functionId <= TSDB_FUNC_TWA) || @@ -2286,7 +2285,7 @@ int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) { return TSDB_CODE_INVALID_SQL; } - tscSqlExprUpdate(pQueryInfo, k, functionId, pExpr->colInfo.colIdx, TSDB_DATA_TYPE_BINARY, bytes); + tscSqlExprUpdate(pQueryInfo, k, functionId, pExpr->colInfo.colIndex, TSDB_DATA_TYPE_BINARY, bytes); // todo refactor pExpr->interResBytes = intermediateBytes; } @@ -2305,7 +2304,7 @@ void tscRestoreSQLFunctionForMetricQuery(SQueryInfo* pQueryInfo) { for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIdx); + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex); // if (/*(pExpr->functionId >= TSDB_FUNC_FIRST_DST && pExpr->functionId <= TSDB_FUNC_LAST_DST) || // (pExpr->functionId >= TSDB_FUNC_SUM && pExpr->functionId <= TSDB_FUNC_MAX) || @@ -2352,7 +2351,7 @@ bool hasUnsupportFunctionsForSTableQuery(SQueryInfo* pQueryInfo) { } if (pQueryInfo->groupbyExpr.numOfGroupCols != 1 || - pQueryInfo->groupbyExpr.columnInfo[0].colIdx != TSDB_TBNAME_COLUMN_INDEX) { + pQueryInfo->groupbyExpr.columnInfo[0].colIndex != TSDB_TBNAME_COLUMN_INDEX) { invalidSqlErrMsg(pQueryInfo->msg, msg2); return true; } @@ -2403,12 +2402,12 @@ void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex) { */ if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 && pQueryInfo->groupbyExpr.tableIndex == tableIndex) { for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { - int32_t index = pQueryInfo->groupbyExpr.columnInfo[i].colIdx; + int32_t index = pQueryInfo->groupbyExpr.columnInfo[i].colIndex; for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) { int32_t tagColIndex = pTableMetaInfo->tagColumnIndex[j]; if (tagColIndex == index) { - pQueryInfo->groupbyExpr.columnInfo[i].colIdx = j; + pQueryInfo->groupbyExpr.columnInfo[i].colIndex = j; break; } } @@ -2429,8 +2428,8 @@ void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex) { } for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) { - if (pExpr->colInfo.colIdx == pTableMetaInfo->tagColumnIndex[j]) { - pExpr->colInfo.colIdx = j; + if (pExpr->colInfo.colIndex == pTableMetaInfo->tagColumnIndex[j]) { + pExpr->colInfo.colIndex = j; break; } } @@ -2529,8 +2528,8 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* } pQueryInfo->groupbyExpr.columnInfo[i] = - (SColIndexEx){.colIdx = relIndex, .flag = TSDB_COL_TAG, .colId = pSchema->colId}; // relIndex; - addRequiredTagColumn(pQueryInfo, pQueryInfo->groupbyExpr.columnInfo[i].colIdx, index.tableIndex); + (SColIndex){.colIndex = relIndex, .flag = TSDB_COL_TAG, .colId = pSchema->colId}; // relIndex; + addRequiredTagColumn(pQueryInfo, pQueryInfo->groupbyExpr.columnInfo[i].colIndex, index.tableIndex); } else { // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by if (pSchema->type > TSDB_DATA_TYPE_BINARY) { @@ -2539,8 +2538,8 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* tscColumnBaseInfoInsert(pQueryInfo, &index); pQueryInfo->groupbyExpr.columnInfo[i] = - (SColIndexEx){.colIdx = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId}; // relIndex; - pQueryInfo->groupbyExpr.orderType = TSQL_SO_ASC; + (SColIndex){.colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId}; // relIndex; + pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; if (i == 0 && pList->nExpr > 1) { return invalidSqlErrMsg(pQueryInfo->msg, msg7); @@ -2630,10 +2629,10 @@ static int32_t doExtractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnFilterIn pColumnFilter->upperRelOptr = TSDB_RELATION_LESS; break; case TK_GT: - pColumnFilter->lowerRelOptr = TSDB_RELATION_LARGE; + pColumnFilter->lowerRelOptr = TSDB_RELATION_GREATER; break; case TK_GE: - pColumnFilter->lowerRelOptr = TSDB_RELATION_LARGE_EQUAL; + pColumnFilter->lowerRelOptr = TSDB_RELATION_GREATER_EQUAL; break; case TK_EQ: pColumnFilter->lowerRelOptr = TSDB_RELATION_EQUAL; @@ -2892,6 +2891,7 @@ static void relToString(tSQLExpr* pExpr, char** str) { } } +UNUSED_FUNC static int32_t getTagCondString(tSQLExpr* pExpr, char** str) { if (pExpr == NULL) { return TSDB_CODE_SUCCESS; @@ -3593,7 +3593,7 @@ static bool validateFilterExpr(SQueryInfo* pQueryInfo) { int32_t lowerOptr = pColFilter->lowerRelOptr; int32_t upperOptr = pColFilter->upperRelOptr; - if ((lowerOptr == TSDB_RELATION_LARGE_EQUAL || lowerOptr == TSDB_RELATION_LARGE) && + if ((lowerOptr == TSDB_RELATION_GREATER_EQUAL || lowerOptr == TSDB_RELATION_GREATER) && (upperOptr == TSDB_RELATION_LESS_EQUAL || upperOptr == TSDB_RELATION_LESS)) { continue; } @@ -3727,28 +3727,27 @@ static void doAddJoinTagsColumnsIntoTagList(SQueryInfo* pQueryInfo, SCondExpr* p static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, tSQLExpr** pExpr) { int32_t ret = TSDB_CODE_SUCCESS; - if (pCondExpr->pTagCond != NULL) { - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - tSQLExpr* p1 = extractExprForSTable(pExpr, pQueryInfo, i); + if (pCondExpr->pTagCond == NULL) { + return ret; + } + + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + tSQLExpr* p1 = extractExprForSTable(pExpr, pQueryInfo, i); + tExprNode* p = NULL; + + ret = exprTreeFromSqlExpr(&p, p1, NULL, pQueryInfo, NULL); + SBuffer buf = exprTreeToBinary(p); + + int64_t uid = tscGetMetaInfo(pQueryInfo, i)->pTableMeta->uid; + tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &buf); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - - char c[TSDB_MAX_TAGS_LEN] = {0}; - char* str = c; - - if ((ret = getTagCondString(p1, &str)) != TSDB_CODE_SUCCESS) { - return ret; - } - - tsSetSTableQueryCond(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid, c); - - doCompactQueryExpr(pExpr); - tSQLExprDestroy(p1); - } - - pCondExpr->pTagCond = NULL; + doCompactQueryExpr(pExpr); + + tSQLExprDestroy(p1); + tExprTreeDestroy(&p, NULL); } + pCondExpr->pTagCond = NULL; return ret; } int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql) { @@ -4044,11 +4043,11 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) { static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { /* set default timestamp order information for all queries */ - pQueryInfo->order.order = TSQL_SO_ASC; + pQueryInfo->order.order = TSDB_ORDER_ASC; STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (isTopBottomQuery(pQueryInfo)) { - pQueryInfo->order.order = TSQL_SO_ASC; + pQueryInfo->order.order = TSDB_ORDER_ASC; pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; } else { pQueryInfo->order.orderColId = -1; @@ -4056,7 +4055,7 @@ static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { /* for metric query, set default ascending order for group output */ if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - pQueryInfo->groupbyExpr.orderType = TSQL_SO_ASC; + pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; } } @@ -4112,7 +4111,7 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { int32_t relTagIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - if (relTagIndex == pQueryInfo->groupbyExpr.columnInfo[0].colIdx) { + if (relTagIndex == pQueryInfo->groupbyExpr.columnInfo[0].colIndex) { orderByTags = true; } } else if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { @@ -4139,7 +4138,7 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema assert(pExpr->functionId == TSDB_FUNC_TS); pExpr = tscSqlExprGet(pQueryInfo, 1); - if (pExpr->colInfo.colIdx != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (pExpr->colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } @@ -4190,7 +4189,7 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema assert(pExpr->functionId == TSDB_FUNC_TS); pExpr = tscSqlExprGet(pQueryInfo, 1); - if (pExpr->colInfo.colIdx != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (pExpr->colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } @@ -4705,7 +4704,7 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* // filter the query functions operating on "tbname" column that are not supported by normal columns. for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - if (pExpr->colInfo.colIdx == TSDB_TBNAME_COLUMN_INDEX) { + if (pExpr->colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } } @@ -4851,10 +4850,10 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau int32_t relIndex = index.columnIndex; - pExpr->colInfo.colIdx = relIndex; - pQueryInfo->groupbyExpr.columnInfo[0].colIdx = relIndex; + pExpr->colInfo.colIndex = relIndex; + pQueryInfo->groupbyExpr.columnInfo[0].colIndex = relIndex; - addRequiredTagColumn(pQueryInfo, pQueryInfo->groupbyExpr.columnInfo[0].colIdx, 0); + addRequiredTagColumn(pQueryInfo, pQueryInfo->groupbyExpr.columnInfo[0].colIndex, 0); } } } @@ -4867,7 +4866,7 @@ static void doLimitOutputNormalColOfGroupby(SSqlExpr* pExpr) { } void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { - int32_t index = pQueryInfo->groupbyExpr.columnInfo[tagIndex].colIdx; + int32_t index = pQueryInfo->groupbyExpr.columnInfo[tagIndex].colIndex; STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -4909,7 +4908,7 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId != TSDB_FUNC_TAG_DUMMY && pExpr->functionId != TSDB_FUNC_TS_DUMMY) { - SSchema* pColSchema = &pSchema[pExpr->colInfo.colIdx]; + SSchema* pColSchema = &pSchema[pExpr->colInfo.colIndex]; getResultDataInfo(pColSchema->type, pColSchema->bytes, pExpr->functionId, pExpr->param[0].i64Key, &pExpr->resType, &pExpr->resBytes, &pExpr->interResBytes, tagLength, true); } @@ -5087,16 +5086,16 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) { char* name = NULL; for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { - SColIndexEx* pColIndex = &pQueryInfo->groupbyExpr.columnInfo[i]; + SColIndex* pColIndex = &pQueryInfo->groupbyExpr.columnInfo[i]; - int16_t colIndex = pColIndex->colIdx; - if (pColIndex->colIdx == TSDB_TBNAME_COLUMN_INDEX) { + int16_t colIndex = pColIndex->colIndex; + if (pColIndex->colIndex == TSDB_TBNAME_COLUMN_INDEX) { type = TSDB_DATA_TYPE_BINARY; bytes = TSDB_TABLE_NAME_LEN; name = TSQL_TBNAME_L; } else { - colIndex = (TSDB_COL_IS_TAG(pColIndex->flag)) ? tscGetNumOfColumns(pTableMetaInfo->pTableMeta) + pColIndex->colIdx - : pColIndex->colIdx; + colIndex = (TSDB_COL_IS_TAG(pColIndex->flag)) ? tscGetNumOfColumns(pTableMetaInfo->pTableMeta) + pColIndex->colIndex + : pColIndex->colIndex; type = pSchema[colIndex].type; bytes = pSchema[colIndex].bytes; @@ -5175,7 +5174,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { if (functId == TSDB_FUNC_PRJ && pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { bool qualified = false; for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { - SColIndexEx* pColIndex = &pQueryInfo->groupbyExpr.columnInfo[j]; + SColIndex* pColIndex = &pQueryInfo->groupbyExpr.columnInfo[j]; if (pColIndex->colId == pExpr->colInfo.colId) { qualified = true; break; @@ -5192,7 +5191,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } - if (functId == TSDB_FUNC_COUNT && pExpr->colInfo.colIdx == TSDB_TBNAME_COLUMN_INDEX) { + if (functId == TSDB_FUNC_COUNT && pExpr->colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } } @@ -5807,37 +5806,38 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_SUCCESS; // Does not build query message here } -static int32_t convertSyntaxTreeToExprTree(tExprNode **pExpr, tSQLExpr* pAst, int32_t* num, - SColIndexEx** pColIndex, SSqlExprInfo* pExprInfo) { +int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SSqlExprInfo* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols) { tExprNode* pLeft = NULL; tExprNode* pRight= NULL; - if (pAst->pLeft != NULL) { - int32_t ret = convertSyntaxTreeToExprTree(&pLeft, pAst->pLeft, num, pColIndex, pExprInfo); + if (pSqlExpr->pLeft != NULL) { + int32_t ret = exprTreeFromSqlExpr(&pLeft, pSqlExpr->pLeft, pExprInfo, pQueryInfo, pCols); if (ret != TSDB_CODE_SUCCESS) { return ret; } } - if (pAst->pRight != NULL) { - int32_t ret = convertSyntaxTreeToExprTree(&pRight, pAst->pRight, num, pColIndex, pExprInfo); + if (pSqlExpr->pRight != NULL) { + int32_t ret = exprTreeFromSqlExpr(&pRight, pSqlExpr->pRight, pExprInfo, pQueryInfo, pCols); if (ret != TSDB_CODE_SUCCESS) { return ret; } } - if (pAst->pLeft == NULL) { - if (pAst->nSQLOptr >= TK_TINYINT && pAst->nSQLOptr <= TK_DOUBLE) { - *pExpr = calloc(1, sizeof(tExprNode) + sizeof(tVariant)); + if (pSqlExpr->pLeft == NULL) { + if (pSqlExpr->nSQLOptr >= TK_TINYINT && pSqlExpr->nSQLOptr <= TK_DOUBLE) { + *pExpr = calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TSQL_NODE_VALUE; - (*pExpr)->pVal = (tVariant*) ((char*)(*pExpr) + sizeof(tExprNode)); - tVariantAssign((*pExpr)->pVal, &pAst->val); - } else if (pAst->nSQLOptr >= TK_COUNT && pAst->nSQLOptr <= TK_AVG_IRATE) { - *pExpr = calloc(1, sizeof(tExprNode) + sizeof(SSchemaEx)); + (*pExpr)->pVal = calloc(1, sizeof(tVariant)); + + tVariantAssign((*pExpr)->pVal, &pSqlExpr->val); + + } else if (pSqlExpr->nSQLOptr >= TK_COUNT && pSqlExpr->nSQLOptr <= TK_AVG_IRATE) { + *pExpr = calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TSQL_NODE_COL; - (*pExpr)->pSchema = (SSchema*)((char*)(*pExpr) + sizeof(tExprNode)); - strncpy((*pExpr)->pSchema->name, pAst->operand.z, pAst->operand.n); - + (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); + strncpy((*pExpr)->pSchema->name, pSqlExpr->operand.z, pSqlExpr->operand.n); + // set the input column data byte and type. for (int32_t i = 0; i < pExprInfo->numOfExprs; ++i) { if (strcmp((*pExpr)->pSchema->name, pExprInfo->pExprs[i]->aliasName) == 0) { @@ -5846,22 +5846,43 @@ static int32_t convertSyntaxTreeToExprTree(tExprNode **pExpr, tSQLExpr* pAst, in break; } } - } else { //todo return error + } else if (pSqlExpr->nSQLOptr == TK_ID) { // column name + SColumnIndex index = {0}; + int32_t ret = getColumnIndexByName(&pSqlExpr->colInfo, pQueryInfo, &index); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + *pExpr = calloc(1, sizeof(tExprNode)); + (*pExpr)->nodeType = TSQL_NODE_COL; + (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); + + STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); + *(*pExpr)->pSchema = *pSchema; + return TSDB_CODE_SUCCESS; + } else { + return TSDB_CODE_INVALID_SQL; } - *pColIndex = realloc(*pColIndex, (++(*num)) * sizeof(SColIndexEx)); - memset(&(*pColIndex)[(*num) - 1], 0, sizeof(SColIndexEx)); + if (pCols != NULL) { // record the involved columns + SColIndex colIndex = {0}; + strncpy(colIndex.name, pSqlExpr->operand.z, pSqlExpr->operand.n); + taosArrayPush(pCols, &colIndex); + } - strncpy((*pColIndex)[(*num) - 1].name, pAst->operand.z, pAst->operand.n); } else { *pExpr = (tExprNode *)calloc(1, sizeof(tExprNode)); + (*pExpr)->nodeType = TSQL_NODE_EXPR; + (*pExpr)->_node.hasPK = false; (*pExpr)->_node.pLeft = pLeft; (*pExpr)->_node.pRight = pRight; - SSQLToken t = {.type = pAst->nSQLOptr}; + + SSQLToken t = {.type = pSqlExpr->nSQLOptr}; (*pExpr)->_node.optr = getBinaryExprOptr(&t); - + assert((*pExpr)->_node.optr != 0); if ((*pExpr)->_node.optr == TSDB_BINARY_OP_DIVIDE) { diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index 1d4ed51f32..d1ec5088d3 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -45,7 +45,7 @@ int32_t treeComparator(const void *pLeft, const void *pRight, void *param) { return -1; } - if (pParam->groupOrderType == TSQL_SO_DESC) { // desc + if (pParam->groupOrderType == TSDB_ORDER_DESC) { // desc return compare_d(pDesc, pParam->numOfElems, pLocalData[pLeftIdx]->rowIdx, pLocalData[pLeftIdx]->filePage.data, pParam->numOfElems, pLocalData[pRightIdx]->rowIdx, pLocalData[pRightIdx]->filePage.data); } else { @@ -652,7 +652,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); - SSchema *p1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIdx); + SSchema *p1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex); int16_t inter = 0; int16_t type = -1; @@ -990,7 +990,7 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, pInterpoInfo); } - if (pQueryInfo->order.order == TSQL_SO_ASC) { + if (pQueryInfo->order.order == TSDB_ORDER_ASC) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); @@ -1168,7 +1168,7 @@ bool needToMerge(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer, tFilePage } else { tOrderDescriptor *pDesc = pLocalReducer->pDesc; if (pDesc->orderIdx.numOfCols > 0) { - if (pDesc->tsOrder == TSQL_SO_ASC) { // asc + if (pDesc->tsOrder == TSDB_ORDER_ASC) { // asc // todo refactor comparator ret = compare_a(pLocalReducer->pDesc, 1, 0, pLocalReducer->prevRowOfInput, 1, 0, tmpBuffer->data); } else { // desc diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index a6c26d4b8f..1492fc9847 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -37,12 +37,25 @@ int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; int (*tscProcessMsgRsp[TSDB_SQL_MAX])(SSqlObj *pSql); void tscProcessActivityTimer(void *handle, void *tmrId); int tscKeepConn[TSDB_SQL_MAX] = {0}; + TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid); void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts); void tscSaveSubscriptionProgress(void* sub); static int32_t minMsgSize() { return tsRpcHeadSize + 100; } +static void tscSetDnodeIpList(SSqlObj* pSql, STableMeta* pTableMeta) { + SRpcIpSet* pIpList = &pSql->ipList; + + pIpList->numOfIps = pTableMeta->numOfVpeers; + pIpList->port = tsDnodeShellPort; + pIpList->inUse = 0; + + for(int32_t i = 0; i < pTableMeta->numOfVpeers; ++i) { + pIpList->ip[i] = pTableMeta->vpeerDesc[i].ip; + } +} + void tscPrintMgmtIp() { if (tscMgmtIpList.numOfIps <= 0) { tscError("invalid mgmt IP list:%d", tscMgmtIpList.numOfIps); @@ -178,17 +191,6 @@ int tscSendMsgToServer(SSqlObj *pSql) { } if (pSql->cmd.command < TSDB_SQL_MGMT) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - pSql->ipList.numOfIps = pTableMeta->numOfVpeers; - pSql->ipList.port = tsDnodeShellPort; - pSql->ipList.inUse = 0; - - for(int32_t i = 0; i < pTableMeta->numOfVpeers; ++i) { - pSql->ipList.ip[i] = pTableMeta->vpeerDesc[i].ip; - } - tscPrint("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port); memcpy(pMsg, pSql->cmd.payload + tsRpcHeadSize, pSql->cmd.payloadLen); @@ -266,19 +268,18 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { rpcFreeCont(rpcMsg->pCont); return; } else { - tscTrace("%p it shall renew table meta, code:%d", pSql, tstrerror(rpcMsg->code)); + tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry); - pSql->maxRetry = TSDB_VNODES_SUPPORT * 2; + pSql->maxRetry = TSDB_VNODES_SUPPORT * 2; // todo move away pSql->res.code = rpcMsg->code; // keep the previous error code - if (++pSql->retry > pSql->maxRetry) { - tscError("%p max retry %d reached, ", pSql, pSql->retry); - return; - } - - rpcMsg->code = tscRenewMeterMeta(pSql, pTableMetaInfo->name); - - if (pTableMetaInfo->pTableMeta) { - tscSendMsgToServer(pSql); + if (pSql->retry > pSql->maxRetry) { + tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry); + } else { + rpcMsg->code = tscRenewMeterMeta(pSql, pTableMetaInfo->name); + if (pTableMetaInfo->pTableMeta) { + tscSendMsgToServer(pSql); + } + rpcFreeCont(rpcMsg->pCont); return; } @@ -286,7 +287,11 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { } } - pSql->retry = 0; + if (pRes->code == TSDB_CODE_SUCCESS) { + tscTrace("%p reset retry counter to be 0 due to success rsp, old:%d", pSql, pSql->retry); + pSql->retry = 0; + } + pRes->rspLen = 0; if (pRes->code != TSDB_CODE_QUERY_CANCELLED) { @@ -331,7 +336,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { pMsg->numOfFailedBlocks = htonl(pMsg->numOfFailedBlocks); pRes->numOfRows += pMsg->affectedRows; - tscTrace("%p cmd:%d code:%d, inserted rows:%d, rsp len:%d", pSql, pCmd->command, pRes->code, + tscTrace("%p cmd:%d code:%s, inserted rows:%d, rsp len:%d", pSql, pCmd->command, tstrerror(pRes->code), pMsg->affectedRows, pRes->rspLen); } else { tscTrace("%p cmd:%d code:%s rsp len:%d", pSql, pCmd->command, tstrerror(pRes->code), pRes->rspLen); @@ -550,8 +555,9 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // pSql->cmd.payloadLen is set during copying data into payload pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - tscTrace("%p build submit msg, vgId:%d numOfVnodes:%d", pSql, pTableMeta->vgId, htonl(pMsgDesc->numOfVnodes)); + tscSetDnodeIpList(pSql, pTableMeta); + tscTrace("%p build submit msg, vgId:%d numOfVnodes:%d", pSql, pTableMeta->vgId, htonl(pMsgDesc->numOfVnodes)); return TSDB_CODE_SUCCESS; } @@ -644,15 +650,22 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { numOfTables = 1; + tscSetDnodeIpList(pSql, pTableMeta); pQueryMsg->head.vgId = htonl(pTableMeta->vgId); tscTrace("%p queried tables:%d, table id: %s", pSql, 1, pTableMetaInfo->name); - } else { // query on super table + } else { // query super table if (pTableMetaInfo->vnodeIndex < 0) { tscError("%p error vnodeIdx:%d", pSql, pTableMetaInfo->vnodeIndex); return -1; } - uint32_t vnodeId = 1; + pSql->ipList.numOfIps = taosArrayGetSize(pTableMetaInfo->vgroupIdList); + pSql->ipList.port = tsDnodeShellPort; + pSql->ipList.inUse = 0; + + for(int32_t i = 0; i < pSql->ipList.numOfIps; ++i) { + pSql->ipList.ip[i] = *(uint32_t*) taosArrayGet(pTableMetaInfo->vgroupIdList, i); + } #if 0 SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); @@ -665,12 +678,13 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } #endif + uint32_t vnodeId = 1; tscTrace("%p query on vid:%d, number of tables:%d", pSql, vnodeId, numOfTables); pQueryMsg->head.vgId = htonl(vnodeId); numOfTables = 1; } - if (pQueryInfo->order.order == TSQL_SO_ASC) { + if (pQueryInfo->order.order == TSDB_ORDER_ASC) { pQueryMsg->window.skey = htobe64(pQueryInfo->stime); pQueryMsg->window.ekey = htobe64(pQueryInfo->etime); } else { @@ -760,14 +774,14 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId)) { - /* column id is not valid according to the cached metermeta, the meter meta is expired */ + /* column id is not valid according to the cached metermeta, the table meta is expired */ tscError("%p table schema is not matched with parsed sql", pSql); return -1; } - pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId); - pSqlFuncExpr->colInfo.colIdx = htons(pExpr->colInfo.colIdx); - pSqlFuncExpr->colInfo.flag = htons(pExpr->colInfo.flag); + pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId); + pSqlFuncExpr->colInfo.colIndex = htons(pExpr->colInfo.colIndex); + pSqlFuncExpr->colInfo.flag = htons(pExpr->colInfo.flag); pSqlFuncExpr->functionId = htons(pExpr->functionId); pSqlFuncExpr->numOfParams = htons(pExpr->numOfParams); @@ -815,16 +829,13 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->orderType = htons(pGroupbyExpr->orderType); for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { - SColIndexEx *pCol = &pGroupbyExpr->columnInfo[j]; + SColIndex *pCol = &pGroupbyExpr->columnInfo[j]; *((int16_t *)pMsg) = pCol->colId; pMsg += sizeof(pCol->colId); - *((int16_t *)pMsg) += pCol->colIdx; - pMsg += sizeof(pCol->colIdx); - - *((int16_t *)pMsg) += pCol->colIdxInBuf; - pMsg += sizeof(pCol->colIdxInBuf); + *((int16_t *)pMsg) += pCol->colIndex; + pMsg += sizeof(pCol->colIndex); *((int16_t *)pMsg) += pCol->flag; pMsg += sizeof(pCol->flag); @@ -866,34 +877,19 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } // serialize tag column query condition - if (pQueryInfo->tagCond.numOfTagCond > 0) { + if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) { STagCond* pTagCond = &pQueryInfo->tagCond; - SCond *pCond = tsGetSTableQueryCondPos(pTagCond, pTableMeta->uid); + SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->uid); if (pCond != NULL && pCond->cond != NULL) { - size_t condLen = strlen(pCond->cond) + 1; + pQueryMsg->tagCondLen = htons(pCond->len); + memcpy(pMsg, pCond->cond, pCond->len); - bool ret = taosMbsToUcs4(pCond->cond, condLen, pMsg, condLen * TSDB_NCHAR_SIZE); - if (!ret) { - tscError("%p mbs to ucs4 failed:%d", pSql, tsGetSTableQueryCondPos(pTagCond, pTableMeta->uid)); - return 0; - } - - pQueryMsg->tagCondLen = htons(condLen); - pMsg += condLen * TSDB_NCHAR_SIZE; + pMsg += pCond->len; } } // tbname in/like query expression should be sent to mgmt node - STagCond* pTagCond = &pQueryInfo->tagCond; - if (pTagCond->tbnameCond.cond != NULL) { - size_t s = strlen(pTagCond->tbnameCond.cond); - memcpy(pMsg, pTagCond->tbnameCond.cond, s); - - pQueryMsg->nameCondLen = htons(s); - pMsg += s; - } - msgLen = pMsg - pStart; tscTrace("%p msg built success,len:%d bytes", pSql, msgLen); @@ -1546,7 +1542,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } /** - * multi meter meta req pkg format: + * multi table meta req pkg format: * | SMgmtHead | SCMMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ...... * no used 4B **/ @@ -1591,8 +1587,10 @@ static UNUSED_FUNC int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); int32_t n = 0; - for (int32_t i = 0; i < pQueryInfo->tagCond.numOfTagCond; ++i) { - n += strlen(pQueryInfo->tagCond.cond[i].cond); + size_t size = taosArrayGetSize(pQueryInfo->tagCond.pCond); + for (int32_t i = 0; i < size; ++i) { + assert(0); +// n += strlen(pQueryInfo->tagCond.cond[i].cond); } int32_t tagLen = n * TSDB_NCHAR_SIZE; @@ -1603,7 +1601,7 @@ static UNUSED_FUNC int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) { int32_t joinCondLen = (TSDB_TABLE_ID_LEN + sizeof(int16_t)) * 2; int32_t elemSize = sizeof(SSuperTableMetaElemMsg) * pQueryInfo->numOfTables; - int32_t colSize = pQueryInfo->groupbyExpr.numOfGroupCols*sizeof(SColIndexEx); + int32_t colSize = pQueryInfo->groupbyExpr.numOfGroupCols*sizeof(SColIndex); int32_t len = tagLen + joinCondLen + elemSize + colSize + defaultSize; @@ -1675,13 +1673,13 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // convert to unicode before sending to mnode for metric query int32_t condLen = 0; if (pTagCond->numOfTagCond > 0) { - SCond *pCond = tsGetSTableQueryCondPos(pTagCond, uid); + SCond *pCond = tsGetSTableQueryCond(pTagCond, uid); if (pCond != NULL && pCond->cond != NULL) { condLen = strlen(pCond->cond) + 1; bool ret = taosMbsToUcs4(pCond->cond, condLen, pMsg, condLen * TSDB_NCHAR_SIZE); if (!ret) { - tscError("%p mbs to ucs4 failed:%s", pSql, tsGetSTableQueryCondPos(pTagCond, uid)); + tscError("%p mbs to ucs4 failed:%s", pSql, tsGetSTableQueryCond(pTagCond, uid)); return 0; } } @@ -1728,16 +1726,16 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pElem->groupbyTagColumnList = htonl(offset); for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { - SColIndexEx *pCol = &pQueryInfo->groupbyExpr.columnInfo[j]; - SColIndexEx *pDestCol = (SColIndexEx *)pMsg; + SColIndex *pCol = &pQueryInfo->groupbyExpr.columnInfo[j]; + SColIndex *pDestCol = (SColIndex *)pMsg; pDestCol->colIdxInBuf = 0; - pDestCol->colIdx = htons(pCol->colIdx); + pDestCol->colIndex = htons(pCol->colIndex); pDestCol->colId = htons(pDestCol->colId); pDestCol->flag = htons(pDestCol->flag); strncpy(pDestCol->name, pCol->name, tListLen(pCol->name)); - pMsg += sizeof(SColIndexEx); + pMsg += sizeof(SColIndex); } } } @@ -1897,7 +1895,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { } /** - * multi meter meta rsp pkg format: + * multi table meta rsp pkg format: * | STaosRsp | ieType | SCMMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2 * |...... 1B 1B 4B **/ @@ -2129,7 +2127,8 @@ _error_clean: // todo opt performance for(int32_t i = 0; i < pStableVgroup->numOfDnodes; ++i) { - taosArrayPush(pInfo->vgroupIdList, &pStableVgroup->dnodeIps[i]); + int32_t ip = htonl(pStableVgroup->dnodeIps[i]); + taosArrayPush(pInfo->vgroupIdList, &ip); } return pSql->res.code; @@ -2367,7 +2366,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); if (NULL == pNew) { - tscError("%p malloc failed for new sqlobj to get meter meta", pSql); + tscError("%p malloc failed for new sqlobj to get table meta", pSql); return TSDB_CODE_CLI_OUT_OF_MEMORY; } @@ -2382,7 +2381,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf pNew->cmd.autoCreated = pSql->cmd.autoCreated; // create table if not exists if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { - tscError("%p malloc failed for payload to get meter meta", pSql); + tscError("%p malloc failed for payload to get table meta", pSql); free(pNew); return TSDB_CODE_CLI_OUT_OF_MEMORY; @@ -2464,9 +2463,10 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) { * 2. if get metermeta failed, still get the metermeta */ if (pTableMetaInfo->pTableMeta == NULL || !tscQueryOnMetric(pCmd)) { + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; if (pTableMetaInfo->pTableMeta) { - tscTrace("%p update meter meta, old: numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql, - pTableMetaInfo->numOfTags, pCmd->numOfCols, pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->pTableMeta); + tscTrace("%p update table meta, old: numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql, + tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->uid, pTableMeta); } tscWaitingForCreateTable(pCmd); @@ -2495,7 +2495,6 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { } #if 0 - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { char tagstr[TSDB_MAX_TAGS_LEN + 1] = {0}; @@ -2534,7 +2533,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { STableMetaInfo *pMMInfo = tscGetMetaInfo(pQueryInfo, i); STableMeta *pTableMeta = taosCacheAcquireByName(tscCacheHandle, pMMInfo->name); - tscAddMeterMetaInfo(pNewQueryInfo, pMMInfo->name, pTableMeta, NULL, pMMInfo->numOfTags, pMMInfo->tagColumnIndex); + tscAddTableMetaInfo(pNewQueryInfo, pMMInfo->name, pTableMeta, NULL, pMMInfo->numOfTags, pMMInfo->tagColumnIndex); } if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index f6c1fee633..323c5a2310 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -407,7 +407,7 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) { *rows = pRes->tsrow; - return (pQueryInfo->order.order == TSQL_SO_DESC) ? pRes->numOfRows : -pRes->numOfRows; + return (pQueryInfo->order.order == TSDB_ORDER_DESC) ? pRes->numOfRows : -pRes->numOfRows; } static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) { @@ -502,14 +502,14 @@ static void **doSetResultRowData(SSqlObj *pSql) { } for(int32_t k = 0; k < sas->numOfCols; ++k) { - int32_t columnIndex = sas->pExpr->binExprInfo.pReqColumns[k].colIdxInBuf; + int32_t columnIndex = sas->pExpr->binExprInfo.pReqColumns[k].colIndex; SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, columnIndex); sas->elemSize[k] = pExpr->resBytes; sas->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes; } - tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, 1, pRes->buffer[i], sas, TSQL_SO_ASC, getArithemicInputSrc); + tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, 1, pRes->buffer[i], sas, TSDB_ORDER_ASC, getArithemicInputSrc); pRes->tsrow[i] = pRes->buffer[i]; free(sas); //todo optimization diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index d616a8bffa..89b2abae7b 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -26,7 +26,7 @@ typedef struct SInsertSupporter { static void freeSubqueryObj(SSqlObj* pSql); static bool doCompare(int32_t order, int64_t left, int64_t right) { - if (order == TSQL_SO_ASC) { + if (order == TSDB_ORDER_ASC) { return left < right; } else { return left > right; @@ -136,8 +136,8 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSubquerySupporter* pSuppor * 2. only one element for each tag. */ if (output1->tsOrder == -1) { - output1->tsOrder = TSQL_SO_ASC; - output2->tsOrder = TSQL_SO_ASC; + output1->tsOrder = TSDB_ORDER_ASC; + output2->tsOrder = TSDB_ORDER_ASC; } tsBufFlush(output1); @@ -1005,8 +1005,8 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int32_t numOfSubQueries = taosArrayGetSize(pTableMetaInfo->vgroupIdList); - assert(numOfSubQueries > 0); + pSql->numOfSubs = taosArrayGetSize(pTableMetaInfo->vgroupIdList); + assert(pSql->numOfSubs > 0); int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize); if (ret != 0) { @@ -1017,16 +1017,15 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { return pRes->code; } - pSql->pSubs = calloc(numOfSubQueries, POINTER_BYTES); - pSql->numOfSubs = numOfSubQueries; + pSql->pSubs = calloc(pSql->numOfSubs, POINTER_BYTES); - tscTrace("%p retrieved query data from %d vnode(s)", pSql, numOfSubQueries); + tscTrace("%p retrieved query data from %d vnode(s)", pSql, pSql->numOfSubs); SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); - pState->numOfTotal = numOfSubQueries; + pState->numOfTotal = pSql->numOfSubs; pRes->code = TSDB_CODE_SUCCESS; int32_t i = 0; - for (; i < numOfSubQueries; ++i) { + for (; i < pSql->numOfSubs; ++i) { SRetrieveSupport *trs = (SRetrieveSupport *)calloc(1, sizeof(SRetrieveSupport)); if (trs == NULL) { tscError("%p failed to malloc buffer for SRetrieveSupport, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); @@ -1070,22 +1069,22 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { tscTrace("%p sub:%p create subquery success. orderOfSub:%d", pSql, pNew, trs->subqueryIndex); } - if (i < numOfSubQueries) { + if (i < pSql->numOfSubs) { tscError("%p failed to prepare subquery structure and launch subqueries", pSql); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; - tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, numOfSubQueries); + tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pSql->numOfSubs); doCleanupSubqueries(pSql, i, pState); return pRes->code; // free all allocated resource } if (pRes->code == TSDB_CODE_QUERY_CANCELLED) { - tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, numOfSubQueries); + tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pSql->numOfSubs); doCleanupSubqueries(pSql, i, pState); return pRes->code; } - for(int32_t j = 0; j < numOfSubQueries; ++j) { + for(int32_t j = 0; j < pSql->numOfSubs; ++j) { SSqlObj* pSub = pSql->pSubs[j]; SRetrieveSupport* pSupport = pSub->param; @@ -1449,7 +1448,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { tscTrace("%p sub:%p reach the max retry count,set global code:%d", pParentSql, pSql, code); atomic_val_compare_exchange_32(&pState->code, 0, code); } else { // does not reach the maximum retry count, go on - tscTrace("%p sub:%p failed code:%d, retry:%d", pParentSql, pSql, code, trsupport->numOfRetry); + tscTrace("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); SSqlObj *pNew = tscCreateSqlObjForSubquery(pParentSql, trsupport, pSql); if (pNew == NULL) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index dd0945ab04..2385b4ad82 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -53,7 +53,7 @@ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) { const int32_t maxKeySize = TSDB_MAX_TAGS_LEN; // allowed max key size - SCond* cond = tsGetSTableQueryCondPos(pTagCond, uid); + SCond* cond = tsGetSTableQueryCond(pTagCond, uid); char join[512] = {0}; if (pTagCond->joinInfo.hasJoin) { @@ -92,28 +92,41 @@ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) { free(tmp); } -SCond* tsGetSTableQueryCondPos(STagCond* pTagCond, uint64_t uid) { - for (int32_t i = 0; i < TSDB_MAX_JOIN_TABLE_NUM; ++i) { - if (uid == pTagCond->cond[i].uid) { - return &pTagCond->cond[i]; +SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) { + if (pTagCond->pCond == NULL) { + return NULL; + } + + size_t size = taosArrayGetSize(pTagCond->pCond); + for (int32_t i = 0; i < size; ++i) { + SCond* pCond = taosArrayGet(pTagCond->pCond, i); + + if (uid == pCond->uid) { + return pCond; } } return NULL; } -// todo refactor by using SArray -void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, const char* str) { - size_t len = strlen(str); - if (len == 0) { +void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBuffer* pBuf) { + if (tbufTell(pBuf) == 0) { return; } - - SCond* pDest = &pTagCond->cond[pTagCond->numOfTagCond]; - pDest->uid = uid; - pDest->cond = strdup(str); - - pTagCond->numOfTagCond += 1; + + SCond cond = { + .uid = uid, + .len = tbufTell(pBuf), + .cond = NULL, + }; + + cond.cond = tbufGetData(pBuf, true); + + if (pTagCond->pCond == NULL) { + pTagCond->pCond = taosArrayInit(3, sizeof(SCond)); + } + + taosArrayPush(pTagCond->pCond, &cond); } bool tscQueryOnMetric(SSqlCmd* pCmd) { @@ -1185,7 +1198,7 @@ SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functi } } - pExpr->colInfo.colIdx = pColIndex->columnIndex; + pExpr->colInfo.colIndex = pColIndex->columnIndex; pExpr->resType = type; pExpr->resBytes = size; pExpr->interResBytes = interSize; @@ -1207,7 +1220,7 @@ SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functi pExpr->functionId = functionId; - pExpr->colInfo.colIdx = srcColumnIndex; + pExpr->colInfo.colIndex = srcColumnIndex; pExpr->colInfo.colId = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, srcColumnIndex)->colId; pExpr->resType = type; @@ -1642,26 +1655,46 @@ void tscTagCondCopy(STagCond* dest, const STagCond* src) { dest->tbnameCond.uid = src->tbnameCond.uid; memcpy(&dest->joinInfo, &src->joinInfo, sizeof(SJoinInfo)); - - for (int32_t i = 0; i < src->numOfTagCond; ++i) { - if (src->cond[i].cond != NULL) { - dest->cond[i].cond = strdup(src->cond[i].cond); - } - - dest->cond[i].uid = src->cond[i].uid; - } - dest->relType = src->relType; - dest->numOfTagCond = src->numOfTagCond; + + if (src->pCond == NULL) { + return; + } + + size_t s = taosArrayGetSize(src->pCond); + dest->pCond = taosArrayInit(s, sizeof(SCond)); + + for (int32_t i = 0; i < s; ++i) { + SCond* pCond = taosArrayGet(src->pCond, i); + + SCond c = {0}; + c.len = pCond->len; + c.uid = pCond->uid; + + if (pCond->len > 0) { + assert(pCond->cond != NULL); + c.cond = malloc(c.len); + memcpy(c.cond, pCond->cond, c.len); + } + + taosArrayPush(dest->pCond, &c); + } } -void tscTagCondRelease(STagCond* pCond) { - free(pCond->tbnameCond.cond); - for (int32_t i = 0; i < pCond->numOfTagCond; ++i) { - free(pCond->cond[i].cond); +void tscTagCondRelease(STagCond* pTagCond) { + free(pTagCond->tbnameCond.cond); + + if (pTagCond->pCond != NULL) { + size_t s = taosArrayGetSize(pTagCond->pCond); + for (int32_t i = 0; i < s; ++i) { + SCond* p = taosArrayGet(pTagCond->pCond, i); + tfree(p->cond); + } + + taosArrayDestroy(pTagCond->pCond); } - memset(pCond, 0, sizeof(STagCond)); + memset(pTagCond, 0, sizeof(STagCond)); } void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) { @@ -1674,11 +1707,11 @@ void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) { if (TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - int16_t actualTagIndex = pTableMetaInfo->tagColumnIndex[pExpr->colInfo.colIdx]; + int16_t actualTagIndex = pTableMetaInfo->tagColumnIndex[pExpr->colInfo.colIndex]; pColInfo[i].type = (actualTagIndex != -1) ? pTagSchema[actualTagIndex].type : TSDB_DATA_TYPE_BINARY; } else { - pColInfo[i].type = pSchema[pExpr->colInfo.colIdx].type; + pColInfo[i].type = pSchema[pExpr->colInfo.colIndex].type; } } } @@ -1880,8 +1913,8 @@ void tscFreeSubqueryInfo(SSqlCmd* pCmd) { tfree(pCmd->pQueryInfo); } -STableMetaInfo* tscAddMeterMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, - SSuperTableMeta* pMetricMeta, int16_t numOfTags, int16_t* tags) { +STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, + SArray* vgroupList, int16_t numOfTags, int16_t* tags) { void* pAlloc = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); if (pAlloc == NULL) { return NULL; @@ -1900,6 +1933,10 @@ STableMetaInfo* tscAddMeterMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST pTableMetaInfo->pTableMeta = pTableMeta; pTableMetaInfo->numOfTags = numOfTags; + + if (vgroupList != NULL) { + pTableMetaInfo->vgroupIdList = taosArrayClone(vgroupList); + } if (tags != NULL) { memcpy(pTableMetaInfo->tagColumnIndex, tags, sizeof(pTableMetaInfo->tagColumnIndex[0]) * numOfTags); @@ -1910,7 +1947,7 @@ STableMetaInfo* tscAddMeterMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST } STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo* pQueryInfo) { - return tscAddMeterMetaInfo(pQueryInfo, NULL, NULL, NULL, 0, NULL); + return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, 0, NULL); } void doRemoveMeterMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFromCache) { @@ -1961,14 +1998,14 @@ void tscResetForNextRetrieve(SSqlRes* pRes) { } SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql) { - SSqlCmd* pCmd = &pSql->cmd; - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, tableIndex); - + SSqlCmd* pCmd = &pSql->cmd; SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); if (pNew == NULL) { - tscError("%p new subquery failed, tableIndex:%d, vnodeIndex:%d", pSql, tableIndex, pTableMetaInfo->vnodeIndex); + tscError("%p new subquery failed, tableIndex:%d", pSql, tableIndex); return NULL; } + + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, tableIndex); pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; @@ -2084,12 +2121,12 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void if (pPrevSql == NULL) { STableMeta* pTableMeta = taosCacheAcquireByName(tscCacheHandle, name); - SSuperTableMeta* pMetricMeta = NULL; - if (cmd == TSDB_SQL_SELECT) { - pMetricMeta = taosCacheAcquireByName(tscCacheHandle, key); - } +// SSuperTableMeta* pMetricMeta = NULL; +// if (cmd == TSDB_SQL_SELECT) { +// pMetricMeta = taosCacheAcquireByName(tscCacheHandle, key); +// } - pFinalInfo = tscAddMeterMetaInfo(pNewQueryInfo, name, pTableMeta, pMetricMeta, pTableMetaInfo->numOfTags, + pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupIdList, pTableMetaInfo->numOfTags, pTableMetaInfo->tagColumnIndex); } else { // transfer the ownership of pTableMeta/pMetricMeta to the newly create sql object. // STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); @@ -2097,7 +2134,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void // STableMeta* pPrevMeterMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pTableMeta); // SSuperTableMeta* pPrevMetricMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pMetricMeta); -// pFinalInfo = tscAddMeterMetaInfo(pNewQueryInfo, name, pPrevMeterMeta, pPrevMetricMeta, pTableMetaInfo->numOfTags, +// pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevMeterMeta, pPrevMetricMeta, pTableMetaInfo->numOfTags, // pTableMetaInfo->tagColumnIndex); } diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index d6a9447e3d..de5aa1dc12 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -147,10 +147,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); // TODO: check if below is necessary #define TSDB_RELATION_INVALID 0 #define TSDB_RELATION_LESS 1 -#define TSDB_RELATION_LARGE 2 +#define TSDB_RELATION_GREATER 2 #define TSDB_RELATION_EQUAL 3 #define TSDB_RELATION_LESS_EQUAL 4 -#define TSDB_RELATION_LARGE_EQUAL 5 +#define TSDB_RELATION_GREATER_EQUAL 5 #define TSDB_RELATION_NOT_EQUAL 6 #define TSDB_RELATION_LIKE 7 @@ -303,8 +303,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) #define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) -#define TSQL_SO_ASC 1 -#define TSQL_SO_DESC 0 +#define TSDB_ORDER_ASC 1 +#define TSDB_ORDER_DESC 2 #define TSDB_SESSIONS_PER_VNODE (300) #define TSDB_SESSIONS_PER_DNODE (TSDB_SESSIONS_PER_VNODE * TSDB_MAX_VNODES) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 5550ebf177..3d1b22dbd2 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -358,7 +358,7 @@ typedef struct { int32_t vgId; } SMDDropVnodeMsg; -typedef struct SColIndexEx { +typedef struct SColIndex { int16_t colId; /* * colIdx is the index of column in latest schema of table @@ -368,11 +368,10 @@ typedef struct SColIndexEx { * colIdxInBuf is used to denote the index of column in pQuery->colList, * this value is invalid in client side, as well as in cache block of vnode either. */ - int16_t colIdx; - int16_t colIdxInBuf; + int16_t colIndex; uint16_t flag; // denote if it is a tag or not char name[TSDB_COL_NAME_LEN]; -} SColIndexEx; +} SColIndex; /* sql function msg, to describe the message to vnode about sql function * operations in select clause */ @@ -380,7 +379,7 @@ typedef struct SSqlFuncExprMsg { int16_t functionId; int16_t numOfParams; - SColIndexEx colInfo; + SColIndex colInfo; struct ArgElem { int16_t argType; int16_t argBytes; @@ -395,7 +394,7 @@ typedef struct SSqlFuncExprMsg { typedef struct SSqlBinaryExprInfo { struct tExprNode *pBinExpr; /* for binary expression */ int32_t numOfCols; /* binary expression involves the readed number of columns*/ - SColIndexEx * pReqColumns; /* source column list */ + SColIndex * pReqColumns; /* source column list */ } SSqlBinaryExprInfo; typedef struct SSqlFunctionExpr { @@ -467,7 +466,6 @@ typedef struct { int64_t slidingTime; // value for sliding window char slidingTimeUnit; // time interval type, for revisement of interval(1d) uint16_t tagCondLen; // tag length in current query - uint16_t nameCondLen; // table name in/like query expression string length int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; int16_t orderType; // used in group by xx order by xxx @@ -670,7 +668,7 @@ typedef struct { typedef struct STableMetaMsg { int32_t contLen; - char tableId[TSDB_TABLE_ID_LEN]; // table id + char tableId[TSDB_TABLE_ID_LEN]; // table id char stableId[TSDB_TABLE_ID_LEN]; // stable name if it is created according to super table uint8_t numOfTags; uint8_t precision; diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index 616a2a46af..72d6f9bf47 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -48,7 +48,7 @@ typedef void (*__do_filter_suppl_fn_t)(void *, void *); */ typedef struct tQueryInfo { int32_t offset; // offset value in tags - int32_t colIdx; // index of column in schema + int32_t colIndex; // index of column in schema uint8_t optr; // expression operator SSchema sch; // schema of tags tVariant q; // query condition value on the specific schema, filter expression @@ -83,7 +83,7 @@ void tSQLBinaryExprToString(tExprNode *pExpr, char *dst, int32_t *len); void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*)); -void tSQLBinaryExprTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param); +void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param); void tSQLBinaryExprCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, char *, int32_t)); @@ -94,7 +94,7 @@ uint8_t getBinaryExprOptr(SSQLToken *pToken); SBuffer exprTreeToBinary(tExprNode* pExprTree); -tExprNode* exprTreeFromBinary(const void* pBuf, size_t size); +int32_t exprTreeFromBinary(const void* pBuf, size_t size, tExprNode** pExprNode); #ifdef __cplusplus } diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 8c8c1a3a12..dc5f2fcb78 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -39,7 +39,7 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int typedef struct SSqlGroupbyExpr { int16_t tableIndex; int16_t numOfGroupCols; - SColIndexEx columnInfo[TSDB_MAX_TAGS]; // group by columns information + SColIndex columnInfo[TSDB_MAX_TAGS]; // group by columns information int16_t orderIndex; // order by column index int16_t orderType; // order by type: asc/desc } SSqlGroupbyExpr; @@ -63,7 +63,7 @@ typedef struct SWindowResult { typedef struct SResultRec { int64_t total; // total generated result size in rows - int64_t size; // current result set size in rows + int64_t rows; // current result set size in rows int64_t capacity; // capacity of current result output buffer // result size threshold in rows. If the result buffer is larger than this, pause query and return to client @@ -125,7 +125,7 @@ typedef struct SQuery { int8_t precision; int16_t numOfOutputCols; int16_t interpoType; - int16_t checkBufferInLoop; // check if the buffer is full during scan each block + int16_t checkBuffer; // check if the buffer is full during scan each block SLimitVal limit; int32_t rowSize; SSqlGroupbyExpr* pGroupbyExpr; diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 585aa8a9cf..87b974b9ba 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -466,9 +466,9 @@ item(A) ::= ids(X) cpxName(Y). { } %type sortorder {int} -sortorder(A) ::= ASC. {A = TSQL_SO_ASC; } -sortorder(A) ::= DESC. {A = TSQL_SO_DESC;} -sortorder(A) ::= . {A = TSQL_SO_ASC;} //default is descend order +sortorder(A) ::= ASC. {A = TSDB_ORDER_ASC; } +sortorder(A) ::= DESC. {A = TSDB_ORDER_DESC;} +sortorder(A) ::= . {A = TSDB_ORDER_ASC;} //default is descend order //group by clause %type groupby_opt {tVariantList*} diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index c7aa57200c..7514f1e44c 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -102,7 +102,7 @@ extern "C" { #define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 -#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSQL_SO_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) +#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) #define MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY 10000000 #define TOP_BOTTOM_QUERY_LIMIT 100 diff --git a/src/query/src/qast.c b/src/query/src/qast.c index 6674dc1aba..ffc47d2abe 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -18,6 +18,7 @@ #include "tutil.h" #include "tbuffer.h" #include "qast.h" +#include "tcompare.h" #include "qsqlparser.h" #include "qsyntaxtreefunction.h" #include "taosdef.h" @@ -79,22 +80,22 @@ static void reviseBinaryExprIfNecessary(tExprNode **pLeft, tExprNode **pRight, u * switch left and left and right hand side in expr if possible */ if ((*pLeft)->nodeType == TSQL_NODE_VALUE && (*pRight)->nodeType == TSQL_NODE_COL) { - if (*optr >= TSDB_RELATION_LARGE && *optr <= TSDB_RELATION_LARGE_EQUAL && *optr != TSDB_RELATION_EQUAL) { + if (*optr >= TSDB_RELATION_GREATER && *optr <= TSDB_RELATION_GREATER_EQUAL && *optr != TSDB_RELATION_EQUAL) { SWAP(*pLeft, *pRight, tExprNode *); } switch (*optr) { - case TSDB_RELATION_LARGE: + case TSDB_RELATION_GREATER: (*optr) = TSDB_RELATION_LESS; break; case TSDB_RELATION_LESS: - (*optr) = TSDB_RELATION_LARGE; + (*optr) = TSDB_RELATION_GREATER; break; - case TSDB_RELATION_LARGE_EQUAL: + case TSDB_RELATION_GREATER_EQUAL: (*optr) = TSDB_RELATION_LESS_EQUAL; break; case TSDB_RELATION_LESS_EQUAL: - (*optr) = TSDB_RELATION_LARGE_EQUAL; + (*optr) = TSDB_RELATION_GREATER_EQUAL; break; default:; // for other type of operations, do nothing @@ -163,9 +164,9 @@ uint8_t getBinaryExprOptr(SSQLToken *pToken) { case TK_LE: return TSDB_RELATION_LESS_EQUAL; case TK_GT: - return TSDB_RELATION_LARGE; + return TSDB_RELATION_GREATER; case TK_GE: - return TSDB_RELATION_LARGE_EQUAL; + return TSDB_RELATION_GREATER_EQUAL; case TK_NE: return TSDB_RELATION_NOT_EQUAL; case TK_AND: @@ -376,12 +377,12 @@ static char *tSQLOptrToString(uint8_t optr, char *dst) { dst += 1; break; } - case TSDB_RELATION_LARGE: { + case TSDB_RELATION_GREATER: { *dst = '>'; dst += 1; break; } - case TSDB_RELATION_LARGE_EQUAL: { + case TSDB_RELATION_GREATER_EQUAL: { *dst = '>'; *(dst + 1) = '='; dst += 2; @@ -466,8 +467,18 @@ void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { *pExpr = NULL; } +typedef struct { + tVariant v; + int32_t optr; +} SEndPoint; + +typedef struct { + SEndPoint* start; + SEndPoint* end; +} SQueryCond; + //static void setInitialValueForRangeQueryCondition(tSKipListQueryCond *q, int8_t type) { -// q->lowerBndRelOptr = TSDB_RELATION_LARGE; +// q->lowerBndRelOptr = TSDB_RELATION_GREATER; // q->upperBndRelOptr = TSDB_RELATION_LESS; // // switch (type) { @@ -504,60 +515,100 @@ void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { // } //} -//static void tSQLDoFilterInitialResult(tSkipList *pSkipList, bool (*fp)(), tQueryInfo *queryColInfo, -// tQueryResultset *result) { -// // primary key filter, search according to skiplist -// if (queryColInfo->colIdx == 0 && queryColInfo->optr != TSDB_RELATION_LIKE) { -// tSKipListQueryCond q; -// setInitialValueForRangeQueryCondition(&q, queryColInfo->q.nType); -// -// switch (queryColInfo->optr) { -// case TSDB_RELATION_EQUAL: { -// result->num = -// tSkipListPointQuery(pSkipList, &queryColInfo->q, 1, INCLUDE_POINT_QUERY, (tSkipListNode ***)&result->pRes); -// break; -// } -// case TSDB_RELATION_NOT_EQUAL: { -// result->num = -// tSkipListPointQuery(pSkipList, &queryColInfo->q, 1, EXCLUDE_POINT_QUERY, (tSkipListNode ***)&result->pRes); -// break; -// } -// case TSDB_RELATION_LESS_EQUAL: { -// tVariantAssign(&q.upperBnd, &queryColInfo->q); -// q.upperBndRelOptr = queryColInfo->optr; -// result->num = tSkipListQuery(pSkipList, &q, (tSkipListNode ***)&result->pRes); -// break; -// } -// case TSDB_RELATION_LESS: { -// tVariantAssign(&q.upperBnd, &queryColInfo->q); -// result->num = tSkipListQuery(pSkipList, &q, (tSkipListNode ***)&result->pRes); -// break; -// } -// case TSDB_RELATION_LARGE: { -// tVariantAssign(&q.lowerBnd, &queryColInfo->q); -// result->num = tSkipListQuery(pSkipList, &q, (tSkipListNode ***)&result->pRes); -// break; -// } -// case TSDB_RELATION_LARGE_EQUAL: { -// tVariantAssign(&q.lowerBnd, &queryColInfo->q); -// q.lowerBndRelOptr = queryColInfo->optr; -// result->num = tSkipListQuery(pSkipList, &q, (tSkipListNode ***)&result->pRes); -// break; -// } -// default: -// pTrace("skiplist:%p, unsupport query operator:%d", pSkipList, queryColInfo->optr); -// } -// -// tSkipListDestroyKey(&q.upperBnd); -// tSkipListDestroyKey(&q.lowerBnd); -// } else { -// /* -// * Brutal force scan the whole skiplit to find the appropriate result, -// * since the filter is not applied to indexed column. -// */ -// result->num = tSkipListIterateList(pSkipList, (tSkipListNode ***)&result->pRes, fp, queryColInfo); -// } -//} +// todo check for malloc failure +static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { + int32_t optr = queryColInfo->optr; + + if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || + optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { + pCond->start = calloc(1, sizeof(tVariant)); + + tVariantAssign(&pCond->start->v, &queryColInfo->q); + pCond->start->optr = queryColInfo->optr; + } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { + pCond->end = calloc(1, sizeof(tVariant)); + + tVariantAssign(&pCond->end->v, &queryColInfo->q); + pCond->end->optr = queryColInfo->optr; + } + + return TSDB_CODE_SUCCESS; +} + +static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t type, SArray* result) { + SSkipListIterator* iter = NULL; + + if (pCond->start != NULL) { + iter = tSkipListCreateIterFromVal(pSkipList, (char*) &pCond->start->v.i64Key, type, TSDB_ORDER_ASC); + } else { + iter = tSkipListCreateIterFromVal(pSkipList, (char*) &pCond->end->v.i64Key, type, TSDB_ORDER_DESC); + } + + __compar_fn_t func = getComparFunc(pSkipList->keyInfo.type, type); + + if (pCond->start != NULL) { + int32_t optr = pCond->start->optr; + + if (optr == TSDB_RELATION_EQUAL) { + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + int32_t ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->start->v.i64Key); + if (ret == 0) { + taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + } else { + break; + } + } + } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { + bool comp = true; + int32_t ret = 0; + + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + if (comp) { + ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->start->v.i64Key); + assert(ret <= 0); + } + + if (ret == 0 && optr == TSDB_RELATION_GREATER) { + continue; + } else { + taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + comp = false; + } + } + } else if (optr == TSDB_RELATION_NOT_EQUAL) { + assert(0); + } else { + assert(0); + } + } else { + int32_t optr = pCond->end->optr; + + if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { + bool comp = true; + int32_t ret = 0; + + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + if (comp) { + ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->start->v.i64Key); + assert(ret >= 0); + } + + if (ret == 0 && optr == TSDB_RELATION_LESS) { + continue; + } else { + taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + comp = false; // no need to compare anymore + } + } + } + } +} /* * qsort comparator @@ -670,9 +721,7 @@ static bool filterItem(tExprNode *pExpr, const void *pItem, SBinaryFilterSupp *p tExprNode *pLeft = pExpr->_node.pLeft; tExprNode *pRight = pExpr->_node.pRight; - /* - * non-leaf nodes, recursively traverse the syntax tree in the post-root order - */ + //non-leaf nodes, recursively traverse the expression tree in the post-root order if (pLeft->nodeType == TSQL_NODE_EXPR && pRight->nodeType == TSQL_NODE_EXPR) { if (pExpr->_node.optr == TSDB_RELATION_OR) { // or if (filterItem(pLeft, pItem, param)) { @@ -707,7 +756,7 @@ static bool filterItem(tExprNode *pExpr, const void *pItem, SBinaryFilterSupp *p * @param pSchema tag schemas * @param fp filter callback function */ -static void tSQLBinaryTraverseOnResult(tExprNode *pExpr, SArray *pResult, SBinaryFilterSupp *param) { +static void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SBinaryFilterSupp *param) { size_t size = taosArrayGetSize(pResult); SArray* array = taosArrayInit(size, POINTER_BYTES); @@ -736,12 +785,12 @@ static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSki } // post-root order traverse syntax tree -void tSQLBinaryExprTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param) { +void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param) { if (pExpr == NULL) { return; } - tExprNode *pLeft = pExpr->_node.pLeft; + tExprNode *pLeft = pExpr->_node.pLeft; tExprNode *pRight = pExpr->_node.pRight; // recursive traverse left child branch @@ -755,7 +804,7 @@ void tSQLBinaryExprTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *resu * * if the query is a high selectivity filter, only small portion of meters are retrieved. */ - tSQLBinaryTraverseOnResult(pExpr, result, param); + exprTreeTraverseImpl(pExpr, result, param); } else if (weight == 0) { /** * apply the hierarchical expression to every node in skiplist for find the qualified nodes @@ -766,8 +815,8 @@ void tSQLBinaryExprTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *resu SArray* rLeft = taosArrayInit(10, POINTER_BYTES); SArray* rRight = taosArrayInit(10, POINTER_BYTES); - tSQLBinaryExprTraverse(pLeft, pSkipList, rLeft, param); - tSQLBinaryExprTraverse(pRight, pSkipList, rRight, param); + tExprTreeTraverse(pLeft, pSkipList, rLeft, param); + tExprTreeTraverse(pRight, pSkipList, rRight, param); if (pExpr->_node.optr == TSDB_RELATION_AND) { // CROSS intersect(rLeft, rRight, result); @@ -801,13 +850,13 @@ void tSQLBinaryExprTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *resu assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); // we filter the result based on the skiplist index in the first place - tSQLBinaryExprTraverse(pFirst, pSkipList, result, param); + tExprTreeTraverse(pFirst, pSkipList, result, param); /* * recursively perform the filter operation based on the initial results, - * So, we do not set the skiplist index as a parameter + * So, we do not set the skip list index as a parameter */ - tSQLBinaryExprTraverse(pSecond, NULL, result, param); + tExprTreeTraverse(pSecond, NULL, result, param); } } else { // column project assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); @@ -816,8 +865,19 @@ void tSQLBinaryExprTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *resu if (pSkipList == NULL) { tSQLListTraverseOnResult(pExpr, param->fp, result); } else { -// assert(result->num == 0); -// tSQLDoFilterInitialResult(pSkipList, param->fp, pExpr->info, result); + tQueryInfo *pQueryInfo = pExpr->_node.info; + + if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) { + SQueryCond cond = {0}; + /*int32_t ret = */setQueryCond(pQueryInfo, &cond); + tQueryOnSkipList(pSkipList, &cond, pQueryInfo->q.nType, result); + } else { + /* Brutal force scan the whole skip list to find the appropriate result, + * since the filter is not applied to indexed column. + */ + assert(0); +// result->num = tSkipListIterateList(pSkipList, (tSkipListNode ***)&result->pRes, fp, queryColInfo); + } } } } @@ -1012,11 +1072,10 @@ static void exprTreeFromBinaryImpl(tExprNode** pExprTree, SBuffer* pBuf) { *pExprTree = pExpr; } -tExprNode* exprTreeFromBinary(const void* pBuf, size_t size) { +int32_t exprTreeFromBinary(const void* pBuf, size_t size, tExprNode** pExprNode) { SBuffer rbuf = {0}; /*int32_t code =*/ tbufBeginRead(&rbuf, pBuf, size); - - tExprNode* pExprNode = NULL; - exprTreeFromBinaryImpl(&pExprNode, &rbuf); - return pExprNode; -} \ No newline at end of file + exprTreeFromBinaryImpl(pExprNode, &rbuf); + return TSDB_CODE_SUCCESS; +} + diff --git a/src/query/src/qextbuffer.c b/src/query/src/qextbuffer.c index d71d566999..0195ae8b26 100644 --- a/src/query/src/qextbuffer.c +++ b/src/query/src/qextbuffer.c @@ -345,7 +345,7 @@ static FORCE_INLINE int32_t primaryKeyComparator(int64_t f1, int64_t f2, int32_t return 0; } - if (colIdx == 0 && tsOrder == TSQL_SO_DESC) { // primary column desc order + if (colIdx == 0 && tsOrder == TSDB_ORDER_DESC) { // primary column desc order return (f1 < f2) ? 1 : -1; } else { // asc return (f1 < f2) ? -1 : 1; @@ -628,7 +628,7 @@ static int32_t qsort_call = 0; void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, int32_t orderType) { // short array sort, incur another sort procedure instead of quick sort process - __col_compar_fn_t compareFn = (orderType == TSQL_SO_ASC) ? compare_sa : compare_sd; + __col_compar_fn_t compareFn = (orderType == TSDB_ORDER_ASC) ? compare_sa : compare_sd; if (end - start + 1 <= 8) { tColDataInsertSort(pDescriptor, numOfRows, start, end, data, compareFn); diff --git a/src/query/src/qinterpolation.c b/src/query/src/qinterpolation.c index 1731e16ed8..40fcf63c36 100644 --- a/src/query/src/qinterpolation.c +++ b/src/query/src/qinterpolation.c @@ -20,7 +20,7 @@ #include "taosmsg.h" #include "tsqlfunction.h" -#define INTERPOL_IS_ASC_INTERPOL(interp) ((interp)->order == TSQL_SO_ASC) +#define INTERPOL_IS_ASC_INTERPOL(interp) ((interp)->order == TSDB_ORDER_ASC) int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t timeRange, char intervalTimeUnit, int16_t precision) { if (timeRange == 0) { @@ -96,7 +96,7 @@ void taosInterpoSetStartInfo(SInterpolationInfo* pInterpoInfo, int32_t numOfRawD } TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int32_t timeInterval, int8_t intervalTimeUnit, int8_t precision) { - if (order == TSQL_SO_ASC) { + if (order == TSDB_ORDER_ASC) { return ekey; } else { return taosGetIntervalStartTimestamp(ekey, timeInterval, intervalTimeUnit, precision); diff --git a/src/query/src/qpercentile.c b/src/query/src/qpercentile.c index 3b12dee053..7283a9fd3f 100644 --- a/src/query/src/qpercentile.c +++ b/src/query/src/qpercentile.c @@ -83,7 +83,7 @@ static tFilePage *loadIntoBucketFromDisk(tMemBucket *pMemBucket, int32_t segIdx, pListItem = pListItem->pNext; } - tColDataQSort(pDesc, buffer->numOfElems, 0, buffer->numOfElems - 1, buffer->data, TSQL_SO_ASC); + tColDataQSort(pDesc, buffer->numOfElems, 0, buffer->numOfElems - 1, buffer->data, TSDB_ORDER_ASC); pDesc->pColumnModel->capacity = oldCapacity; // restore value return buffer; diff --git a/src/query/src/qsyntaxtreefunction.c b/src/query/src/qsyntaxtreefunction.c index d21f7dab73..41e84b5ab0 100644 --- a/src/query/src/qsyntaxtreefunction.c +++ b/src/query/src/qsyntaxtreefunction.c @@ -21,8 +21,8 @@ #define ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, op, _res_type, _ord) \ { \ - int32_t i = ((_ord) == TSQL_SO_ASC) ? 0 : MAX(len1, len2) - 1; \ - int32_t step = ((_ord) == TSQL_SO_ASC) ? 1 : -1; \ + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(len1, len2) - 1; \ + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; \ \ if ((len1) == (len2)) { \ for (; i < (len2) && i >= 0; i += step, (out) += 1) { \ @@ -53,8 +53,8 @@ #define ARRAY_LIST_OP_REM(left, right, _left_type, _right_type, len1, len2, out, op, _res_type, _ord) \ { \ - int32_t i = (_ord == TSQL_SO_ASC) ? 0 : MAX(len1, len2) - 1; \ - int32_t step = (_ord == TSQL_SO_ASC) ? 1 : -1; \ + int32_t i = (_ord == TSDB_ORDER_ASC) ? 0 : MAX(len1, len2) - 1; \ + int32_t step = (_ord == TSDB_ORDER_ASC) ? 1 : -1; \ \ if (len1 == (len2)) { \ for (; i >= 0 && i < (len2); i += step, (out) += 1) { \ @@ -107,8 +107,8 @@ void calc_fn_i32_i32_add(void *left, void *right, int32_t numLeft, int32_t numRi int32_t *pRight = (int32_t *)right; double * pOutput = (double *)output; - int32_t i = (order == TSQL_SO_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; + int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; if (numLeft == numRight) { for (; i >= 0 && i < numRight; i += step, pOutput += 1) { @@ -305,8 +305,8 @@ void calc_fn_i32_i32_sub(void *left, void *right, int32_t numLeft, int32_t numRi int32_t *pRight = (int32_t *)right; double * pOutput = (double *)output; - int32_t i = (order == TSQL_SO_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; + int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; if (numLeft == numRight) { for (; i >= 0 && i < numRight; i += step, pOutput += 1) { @@ -516,8 +516,8 @@ void calc_fn_i32_i32_multi(void *left, void *right, int32_t numLeft, int32_t num int32_t *pRight = (int32_t *)right; double * pOutput = (double *)output; - int32_t i = (order == TSQL_SO_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; + int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; if (numLeft == numRight) { for (; i >= 0 && i < numRight; i += step, pOutput += 1) { @@ -714,8 +714,8 @@ void calc_fn_i32_i32_div(void *left, void *right, int32_t numLeft, int32_t numRi int32_t *pRight = (int32_t *)right; double * pOutput = (double *)output; - int32_t i = (order == TSQL_SO_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; + int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; if (numLeft == numRight) { for (; i >= 0 && i < numRight; i += step, pOutput += 1) { @@ -928,8 +928,8 @@ void calc_fn_i32_i32_rem(void *left, void *right, int32_t numLeft, int32_t numRi int32_t *pRight = (int32_t *)right; double * pOutput = (double *)output; - int32_t i = (order == TSQL_SO_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; + int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; if (numLeft == numRight) { for (; i >= 0 && i < numRight; i += step, pOutput += 1) { @@ -986,8 +986,8 @@ void calc_fn_i32_d_rem(void *left, void *right, int32_t numLeft, int32_t numRigh double * pRight = (double *)right; double * pOutput = (double *)output; - int32_t i = (order == TSQL_SO_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; + int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; if (numLeft == numRight) { for (; i >= 0 && i < numRight; i += step, pOutput += 1) { diff --git a/src/query/src/qtsbuf.c b/src/query/src/qtsbuf.c index ea6e6dfdc0..062a8038b2 100644 --- a/src/query/src/qtsbuf.c +++ b/src/query/src/qtsbuf.c @@ -33,11 +33,11 @@ STSBuf* tsBufCreate(bool autoDelete) { } // update the header info - STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = TSQL_SO_ASC}; + STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = TSDB_ORDER_ASC}; STSBufUpdateHeader(pTSBuf, &header); tsBufResetPos(pTSBuf); - pTSBuf->cur.order = TSQL_SO_ASC; + pTSBuf->cur.order = TSDB_ORDER_ASC; pTSBuf->autoDelete = autoDelete; pTSBuf->tsOrder = -1; @@ -88,7 +88,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { // check the ts order pTSBuf->tsOrder = header.tsOrder; - if (pTSBuf->tsOrder != TSQL_SO_ASC && pTSBuf->tsOrder != TSQL_SO_DESC) { + if (pTSBuf->tsOrder != TSDB_ORDER_ASC && pTSBuf->tsOrder != TSDB_ORDER_DESC) { // tscError("invalid order info in buf:%d", pTSBuf->tsOrder); tsBufDestory(pTSBuf); return NULL; @@ -118,7 +118,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { tsBufResetPos(pTSBuf); // ascending by default - pTSBuf->cur.order = TSQL_SO_ASC; + pTSBuf->cur.order = TSDB_ORDER_ASC; pTSBuf->autoDelete = autoDelete; @@ -274,7 +274,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { memset(pBlock, 0, sizeof(STSBlock)); pBlock->payload = tmp; - if (order == TSQL_SO_DESC) { + if (order == TSDB_ORDER_DESC) { /* * set the right position for the reversed traverse, the reversed traverse is started from * the end of each comp data block @@ -303,7 +303,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); // for backwards traverse, set the start position at the end of previous block - if (order == TSQL_SO_DESC) { + if (order == TSDB_ORDER_DESC) { int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag); int64_t r = fseek(pTSBuf->f, -offset, SEEK_CUR); UNUSED(r); @@ -321,9 +321,9 @@ static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { TSKEY lastKey = *(TSKEY*)(ptsData->rawBuf + ptsData->len - TSDB_KEYSIZE); if (lastKey > *(TSKEY*)pData) { - pTSBuf->tsOrder = TSQL_SO_DESC; + pTSBuf->tsOrder = TSDB_ORDER_DESC; } else { - pTSBuf->tsOrder = TSQL_SO_ASC; + pTSBuf->tsOrder = TSDB_ORDER_ASC; } } else if (len > TSDB_KEYSIZE) { // no data in current vnode, more than one ts is added, check the orders @@ -331,9 +331,9 @@ static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { TSKEY k2 = *(TSKEY*)(pData + TSDB_KEYSIZE); if (k1 < k2) { - pTSBuf->tsOrder = TSQL_SO_ASC; + pTSBuf->tsOrder = TSDB_ORDER_ASC; } else if (k1 > k2) { - pTSBuf->tsOrder = TSQL_SO_DESC; + pTSBuf->tsOrder = TSDB_ORDER_DESC; } else { // todo handle error } @@ -432,13 +432,13 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int bool decomp = false; while ((i++) <= blockIndex) { - if (readDataFromDisk(pTSBuf, TSQL_SO_ASC, decomp) == NULL) { + if (readDataFromDisk(pTSBuf, TSDB_ORDER_ASC, decomp) == NULL) { return -1; } } // set the file position to be the end of previous comp block - if (pTSBuf->cur.order == TSQL_SO_DESC) { + if (pTSBuf->cur.order == TSDB_ORDER_DESC) { STSBlock* pBlock = &pTSBuf->block; int32_t compBlockSize = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag); @@ -452,7 +452,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo bool decomp = false; int64_t offset = 0; - if (pTSBuf->cur.order == TSQL_SO_ASC) { + if (pTSBuf->cur.order == TSDB_ORDER_ASC) { offset = pBlockInfo->offset; } else { // reversed traverse starts from the end of block offset = pBlockInfo->offset + pBlockInfo->compLen; @@ -482,8 +482,8 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t vnodeIndex, int32_t blockIndex } STSCursor* pCur = &pTSBuf->cur; - if (pCur->vnodeIndex == vnodeIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSQL_SO_ASC) || - (pCur->blockIndex >= blockIndex && pCur->order == TSQL_SO_DESC))) { + if (pCur->vnodeIndex == vnodeIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) || + (pCur->blockIndex >= blockIndex && pCur->order == TSDB_ORDER_DESC))) { int32_t i = 0; bool decomp = false; int32_t step = abs(blockIndex - pCur->blockIndex); @@ -520,7 +520,7 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t vnodeIndex, int32_t blockIndex pCur->vnodeIndex = vnodeIndex; pCur->blockIndex = blockIndex; - pCur->tsIndex = (pCur->order == TSQL_SO_ASC) ? 0 : pBlock->numOfElem - 1; + pCur->tsIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : pBlock->numOfElem - 1; } STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId) { @@ -555,7 +555,7 @@ bool tsBufNextPos(STSBuf* pTSBuf) { // get the first/last position according to traverse order if (pCur->vnodeIndex == -1) { - if (pCur->order == TSQL_SO_ASC) { + if (pCur->order == TSDB_ORDER_ASC) { tsBufGetBlock(pTSBuf, 0, 0); if (pTSBuf->block.numOfElem == 0) { // the whole list is empty, return @@ -587,20 +587,20 @@ bool tsBufNextPos(STSBuf* pTSBuf) { } } - int32_t step = pCur->order == TSQL_SO_ASC ? 1 : -1; + int32_t step = pCur->order == TSDB_ORDER_ASC ? 1 : -1; while (1) { assert(pTSBuf->tsData.len == pTSBuf->block.numOfElem * TSDB_KEYSIZE); - if ((pCur->order == TSQL_SO_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) || - (pCur->order == TSQL_SO_DESC && pCur->tsIndex <= 0)) { + if ((pCur->order == TSDB_ORDER_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) || + (pCur->order == TSDB_ORDER_DESC && pCur->tsIndex <= 0)) { int32_t vnodeId = pTSBuf->pData[pCur->vnodeIndex].info.vnode; STSVnodeBlockInfo* pBlockInfo = tsBufGetVnodeBlockInfo(pTSBuf, vnodeId); - if (pBlockInfo == NULL || (pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSQL_SO_ASC) || - (pCur->blockIndex <= 0 && pCur->order == TSQL_SO_DESC)) { - if ((pCur->vnodeIndex >= pTSBuf->numOfVnodes - 1 && pCur->order == TSQL_SO_ASC) || - (pCur->vnodeIndex <= 0 && pCur->order == TSQL_SO_DESC)) { + if (pBlockInfo == NULL || (pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSDB_ORDER_ASC) || + (pCur->blockIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { + if ((pCur->vnodeIndex >= pTSBuf->numOfVnodes - 1 && pCur->order == TSDB_ORDER_ASC) || + (pCur->vnodeIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { pCur->vnodeIndex = -1; return false; } @@ -609,7 +609,7 @@ bool tsBufNextPos(STSBuf* pTSBuf) { return false; } - int32_t blockIndex = pCur->order == TSQL_SO_ASC ? 0 : pBlockInfo->numOfBlocks - 1; + int32_t blockIndex = pCur->order == TSDB_ORDER_ASC ? 0 : pBlockInfo->numOfBlocks - 1; tsBufGetBlock(pTSBuf, pCur->vnodeIndex + step, blockIndex); break; @@ -766,7 +766,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ pTSBuf->fileSize += len; pTSBuf->tsOrder = order; - assert(order == TSQL_SO_ASC || order == TSQL_SO_DESC); + assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); STSBufFileHeader header = { .magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = pTSBuf->tsOrder}; @@ -850,7 +850,7 @@ void tsBufDisplay(STSBuf* pTSBuf) { printf("number of vnode:%d\n", pTSBuf->numOfVnodes); int32_t old = pTSBuf->cur.order; - pTSBuf->cur.order = TSQL_SO_ASC; + pTSBuf->cur.order = TSDB_ORDER_ASC; tsBufResetPos(pTSBuf); diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index dfa5dae73a..41941e1ff1 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -53,9 +53,9 @@ /* get the qinfo struct address from the query struct address */ #define GET_COLUMN_BYTES(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdx].info.bytes) + ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].info.bytes) #define GET_COLUMN_TYPE(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdx].info.type) + ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].info.type) typedef struct SPointInterpoSupporter { int32_t numOfCols; @@ -280,6 +280,7 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { } } + assert(maxOutput >= 0); return maxOutput; } @@ -294,13 +295,13 @@ bool isGroupbyNormalCol(SSqlGroupbyExpr *pGroupbyExpr) { } for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { - SColIndexEx *pColIndex = &pGroupbyExpr->columnInfo[i]; + SColIndex *pColIndex = &pGroupbyExpr->columnInfo[i]; if (pColIndex->flag == TSDB_COL_NORMAL) { /* * make sure the normal column locates at the second position if tbname exists in group by clause */ if (pGroupbyExpr->numOfGroupCols > 1) { - assert(pColIndex->colIdx > 0); + assert(pColIndex->colIndex > 0); } return true; @@ -317,7 +318,7 @@ int16_t getGroupbyColumnType(SQuery *pQuery, SSqlGroupbyExpr *pGroupbyExpr) { int16_t type = TSDB_DATA_TYPE_NULL; for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { - SColIndexEx *pColIndex = &pGroupbyExpr->columnInfo[i]; + SColIndex *pColIndex = &pGroupbyExpr->columnInfo[i]; if (pColIndex->flag == TSDB_COL_NORMAL) { colId = pColIndex->colId; break; @@ -362,8 +363,8 @@ bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].pBase.functio bool doRevisedResultsByLimit(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - if ((pQuery->limit.limit > 0) && (pQuery->rec.size + pQuery->rec.size > pQuery->limit.limit)) { - pQuery->rec.size = pQuery->limit.limit - pQuery->rec.size; + if ((pQuery->limit.limit > 0) && (pQuery->rec.rows + pQuery->rec.rows > pQuery->limit.limit)) { + pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.rows; // query completed setQueryStatus(pQuery, QUERY_COMPLETED); @@ -390,7 +391,7 @@ static bool isTopBottomQuery(SQuery *pQuery) { static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, int32_t index) { // for a tag column, no corresponding field info - SColIndexEx *pColIndexEx = &pQuery->pSelectExpr[index].pBase.colInfo; + SColIndex *pColIndexEx = &pQuery->pSelectExpr[index].pBase.colInfo; if (TSDB_COL_IS_TAG(pColIndexEx->flag)) { return NULL; } @@ -418,7 +419,7 @@ static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlo */ static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SDataStatis **pColStatis) { - SColIndexEx* pColIndex = &pQuery->pSelectExpr[col].pBase.colInfo; + SColIndex* pColIndex = &pQuery->pSelectExpr[col].pBase.colInfo; if (TSDB_COL_IS_TAG(pColIndex->flag)) { return false; } @@ -593,7 +594,7 @@ static int32_t getForwardStepsInBlock(int32_t numOfPoints, __block_search_fn_t s int32_t forwardStep = 0; if (endPos >= 0) { - forwardStep = (order == TSQL_SO_ASC) ? (endPos - pos) : (pos - endPos); + forwardStep = (order == TSDB_ORDER_ASC) ? (endPos - pos) : (pos - endPos); assert(forwardStep >= 0); // endPos data is equalled to the key so, we do need to read the element in endPos @@ -670,7 +671,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, bool updateLastKey) { - assert(startPos >= 0 && startPos < pDataBlockInfo->size); + assert(startPos >= 0 && startPos < pDataBlockInfo->rows); int32_t num = -1; int32_t order = pQuery->order.order; @@ -679,7 +680,7 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo if (QUERY_IS_ASC_QUERY(pQuery)) { if (ekey < pDataBlockInfo->window.ekey) { - num = getForwardStepsInBlock(pDataBlockInfo->size, searchFn, ekey, startPos, order, pPrimaryColumn); + num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); if (num == 0) { // no qualified data in current block, do not update the lastKey value assert(ekey < pPrimaryColumn[startPos]); } else { @@ -688,14 +689,14 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo } } } else { - num = pDataBlockInfo->size - startPos; + num = pDataBlockInfo->rows - startPos; if (updateLastKey) { pQuery->lastKey = pDataBlockInfo->window.ekey + step; } } } else { // desc if (ekey > pDataBlockInfo->window.skey) { - num = getForwardStepsInBlock(pDataBlockInfo->size, searchFn, ekey, startPos, order, pPrimaryColumn); + num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); if (num == 0) { // no qualified data in current block, do not update the lastKey value assert(ekey > pPrimaryColumn[startPos]); } else { @@ -788,7 +789,7 @@ static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow } } - int32_t startPos = searchFn((char *)primaryKeys, pDataBlockInfo->size, startKey, pQuery->order.order); + int32_t startPos = searchFn((char *)primaryKeys, pDataBlockInfo->rows, startKey, pQuery->order.order); /* * This time window does not cover any data, try next time window, @@ -851,12 +852,12 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 sas->numOfCols = pQuery->numOfCols; sas->offset = 0; } else { // other type of query function - SColIndexEx *pCol = &pQuery->pSelectExpr[col].pBase.colInfo; + SColIndex *pCol = &pQuery->pSelectExpr[col].pBase.colInfo; if (TSDB_COL_IS_TAG(pCol->flag)) { dataBlock = NULL; } else { /* - * the colIdx is acquired from the first meter of all qualified meters in this vnode during query prepare stage, + * the colIndex is acquired from the first meter of all qualified meters in this vnode during query prepare stage, * the remain meter may not have the required column in cache actually. * So, the validation of required column in cache with the corresponding meter schema is reinforced. */ @@ -888,7 +889,7 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 * @return the incremental number of output value, so it maybe 0 for fixed number of query, * such as count/min/max etc. */ -static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, +static void blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, __block_search_fn_t searchFn, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; @@ -902,9 +903,7 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataSt primaryKeyCol = (TSKEY *)(pColInfo->pData); } - pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : pDataBlockInfo->size - 1; - int64_t prevNumOfRes = getNumOfResult(pRuntimeEnv); - + pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : pDataBlockInfo->rows - 1; SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutputCols, sizeof(SArithmeticSupport)); for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { @@ -913,9 +912,9 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataSt SDataStatis *tpField = NULL; bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField); - char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->size, pDataBlock); + char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->size, functionId, tpField, + setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, tpField, hasNull, &sasArray[k], pRuntimeEnv->scanFlag); } @@ -925,11 +924,9 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataSt TSKEY ts = primaryKeyCol[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - assert(0); - // if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pRuntimeEnv->pTabObj->sid, &win) != - // TSDB_CODE_SUCCESS) { - // return 0; - // } + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->sid, &win) != TSDB_CODE_SUCCESS) { + return 0; + } TSKEY ekey = reviseWindowEkey(pQuery, &win); int32_t forwardStep = @@ -949,10 +946,7 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataSt } // null data, failed to allocate more memory buffer - // int32_t sid = pRuntimeEnv->pTabObj->sid; - int32_t sid = 0; - assert(0); - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, sid, &nextWin) != TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->sid, &nextWin) != TSDB_CODE_SUCCESS) { break; } @@ -960,7 +954,6 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataSt forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, primaryKeyCol, startPos, ekey, searchFn, true); pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, primaryKeyCol); } @@ -978,18 +971,8 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataSt } } } - - /* - * No need to calculate the number of output results for group-by normal columns, interval query - * because the results of group by normal column is put into intermediate buffer. - */ - int32_t num = 0; - if (!isIntervalQuery(pQuery)) { - num = getNumOfResult(pRuntimeEnv) - prevNumOfRes; - } - + tfree(sasArray); - return (int32_t)num; } static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes) { @@ -1138,9 +1121,9 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat SDataStatis *pColStatis = NULL; bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &pColStatis); - char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->size, pDataBlock); + char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->size, functionId, pColStatis, + setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, pColStatis, hasNull, &sasArray[k], pRuntimeEnv->scanFlag); } @@ -1162,14 +1145,14 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat // from bottom to top in asc order if (pRuntimeEnv->pTSBuf != NULL) { SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery); - qTrace("QInfo:%p process data rows, numOfRows:%d, query order:%d, ts comp order:%d", pQInfo, pDataBlockInfo->size, + qTrace("QInfo:%p process data rows, numOfRows:%d, query order:%d, ts comp order:%d", pQInfo, pDataBlockInfo->rows, pQuery->order.order, pRuntimeEnv->pTSBuf->cur.order); } int32_t j = 0; TSKEY lastKey = -1; - for (j = 0; j < pDataBlockInfo->size; ++j) { + for (j = 0; j < pDataBlockInfo->rows; ++j) { int32_t offset = GET_COL_DATA_POS(pQuery, j, step); if (pRuntimeEnv->pTSBuf != NULL) { @@ -1271,7 +1254,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat * pointsOffset is the maximum available space in result buffer update the actual forward step for query that * requires checking buffer during loop */ - if ((pQuery->checkBufferInLoop == 1) && (++numOfRes) >= pQuery->pointsOffset) { + if ((pQuery->checkBuffer == 1) && (++numOfRes) >= pQuery->pointsOffset) { pQuery->lastKey = lastKey + step; assert(0); // *forwardStep = j + 1; @@ -1297,7 +1280,7 @@ static UNUSED_FUNC int32_t reviseForwardSteps(SQueryRuntimeEnv *pRuntimeEnv, int /* * 1. If value filter exists, we try all data in current block, and do not set the QUERY_RESBUF_FULL flag. * - * 2. In case of top/bottom/ts_comp query, the checkBufferInLoop == 1 and pQuery->numOfFilterCols + * 2. In case of top/bottom/ts_comp query, the checkBuffer == 1 and pQuery->numOfFilterCols * may be 0 or not. We do not check the capacity of output buffer, since the filter function will do it. * * 3. In handling the query of secondary query of join, tsBuf servers as a ts filter. @@ -1309,7 +1292,7 @@ static UNUSED_FUNC int32_t reviseForwardSteps(SQueryRuntimeEnv *pRuntimeEnv, int } // current buffer does not have enough space, try in the next loop - if ((pQuery->checkBufferInLoop == 1) && (pQuery->pointsOffset <= forwardStep)) { + if ((pQuery->checkBuffer == 1) && (pQuery->pointsOffset <= forwardStep)) { forwardStep = pQuery->pointsOffset; } @@ -1317,17 +1300,17 @@ static UNUSED_FUNC int32_t reviseForwardSteps(SQueryRuntimeEnv *pRuntimeEnv, int } static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, - SDataStatis *pStatis, __block_search_fn_t searchFn, int32_t *numOfRes, - SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { + SDataStatis *pStatis, __block_search_fn_t searchFn, SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { - *numOfRes = rowwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); + /*numOfRes = */rowwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { - *numOfRes = blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); } - TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; + TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; pQuery->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo); @@ -1339,14 +1322,19 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl setQueryStatus(pQuery, QUERY_COMPLETED); } - assert(*numOfRes >= 0); + int32_t numOfRes = getNumOfResult(pRuntimeEnv); - // check if buffer is large enough for accommodating all qualified points - if (*numOfRes > 0 && pQuery->checkBufferInLoop == 1 && ((*numOfRes) >= pQuery->rec.threshold)) { - setQueryStatus(pQuery, QUERY_RESBUF_FULL); + // update the number of output result + if (numOfRes > 0 && pQuery->checkBuffer == 1) { + assert(numOfRes >= pQuery->rec.rows); + pQuery->rec.rows = numOfRes; + + if (numOfRes >= pQuery->rec.threshold) { + setQueryStatus(pQuery, QUERY_RESBUF_FULL); + } } - return 0; + return numOfRes; } void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size, @@ -1450,8 +1438,7 @@ static void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool i } } -static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel *pTagsSchema, int16_t order, - bool isSTableQuery) { +static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel *pTagsSchema, int16_t order) { dTrace("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv)); SQuery *pQuery = pRuntimeEnv->pQuery; @@ -1465,7 +1452,6 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel pRuntimeEnv->offset[0] = 0; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { SSqlFuncExprMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; -// SColIndexEx * pColIndexEx = &pSqlFuncMsg->colInfo; SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; pCtx->inputType = GET_COLUMN_TYPE(pQuery, i); @@ -1511,10 +1497,10 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel } // set the intermediate result output buffer - setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, isSTableQuery); + setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery); // if it is group by normal column, do not set output buffer, the output buffer is pResult - if (!isGroupbyNormalCol(pQuery->pGroupbyExpr) && !isSTableQuery) { + if (!isGroupbyNormalCol(pQuery->pGroupbyExpr) && !pRuntimeEnv->stableQuery) { resetCtxOutputBuf(pRuntimeEnv); } @@ -1606,7 +1592,7 @@ bool isFixedOutputQuery(SQuery *pQuery) { // ignore the ts_comp function if (i == 0 && pExprMsg->functionId == TSDB_FUNC_PRJ && pExprMsg->numOfParams == 1 && - pExprMsg->colInfo.colIdx == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + pExprMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { continue; } @@ -1683,7 +1669,7 @@ static bool needReverseScan(SQuery *pQuery) { } ///////////////////////////////////////////////////////////////////////////////////////////// -void doGetAlignedIntervalQueryRangeImpl(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, +void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, int64_t *realSkey, int64_t *realEkey, STimeWindow *win) { assert(key >= keyFirst && key <= keyLast && pQuery->slidingTime <= pQuery->intervalTime); @@ -1835,9 +1821,9 @@ bool normalizeUnBoundLastRowQuery(SQInfo *pQInfo, SPointInterpoSupporter *pPoint static void setScanLimitationByResultBuffer(SQuery *pQuery) { if (isTopBottomQuery(pQuery)) { - pQuery->checkBufferInLoop = 0; + pQuery->checkBuffer = 0; } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { - pQuery->checkBufferInLoop = 0; + pQuery->checkBuffer = 0; } else { bool hasMultioutput = false; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { @@ -1852,7 +1838,7 @@ static void setScanLimitationByResultBuffer(SQuery *pQuery) { } } - pQuery->checkBufferInLoop = hasMultioutput ? 1 : 0; + pQuery->checkBuffer = hasMultioutput ? 1 : 0; } } @@ -1902,9 +1888,9 @@ static void changeExecuteScanOrder(SQuery *pQuery, bool metricQuery) { // descending order query for last_row query if (isFirstLastRowQuery(pQuery)) { dTrace("QInfo:%p scan order changed for last_row query, old:%d, new:%d", GET_QINFO_ADDR(pQuery), - pQuery->order.order, TSQL_SO_DESC); + pQuery->order.order, TSDB_ORDER_DESC); - pQuery->order.order = TSQL_SO_DESC; + pQuery->order.order = TSDB_ORDER_DESC; int64_t skey = MIN(pQuery->window.skey, pQuery->window.ekey); int64_t ekey = MAX(pQuery->window.skey, pQuery->window.ekey); @@ -1917,56 +1903,56 @@ static void changeExecuteScanOrder(SQuery *pQuery, bool metricQuery) { if (isPointInterpoQuery(pQuery) && pQuery->intervalTime == 0) { if (!QUERY_IS_ASC_QUERY(pQuery)) { - dTrace(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSQL_SO_ASC, pQuery->window.skey, + dTrace(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); } - pQuery->order.order = TSQL_SO_ASC; + pQuery->order.order = TSDB_ORDER_ASC; return; } if (pQuery->intervalTime == 0) { if (onlyFirstQuery(pQuery)) { if (!QUERY_IS_ASC_QUERY(pQuery)) { - dTrace(msg, GET_QINFO_ADDR(pQuery), "only-first", pQuery->order.order, TSQL_SO_ASC, pQuery->window.skey, + dTrace(msg, GET_QINFO_ADDR(pQuery), "only-first", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); } - pQuery->order.order = TSQL_SO_ASC; + pQuery->order.order = TSDB_ORDER_ASC; } else if (onlyLastQuery(pQuery)) { if (QUERY_IS_ASC_QUERY(pQuery)) { - dTrace(msg, GET_QINFO_ADDR(pQuery), "only-last", pQuery->order.order, TSQL_SO_DESC, pQuery->window.skey, + dTrace(msg, GET_QINFO_ADDR(pQuery), "only-last", pQuery->order.order, TSDB_ORDER_DESC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); } - pQuery->order.order = TSQL_SO_DESC; + pQuery->order.order = TSDB_ORDER_DESC; } } else { // interval query if (metricQuery) { if (onlyFirstQuery(pQuery)) { if (!QUERY_IS_ASC_QUERY(pQuery)) { - dTrace(msg, GET_QINFO_ADDR(pQuery), "only-first stable", pQuery->order.order, TSQL_SO_ASC, + dTrace(msg, GET_QINFO_ADDR(pQuery), "only-first stable", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); } - pQuery->order.order = TSQL_SO_ASC; + pQuery->order.order = TSDB_ORDER_ASC; } else if (onlyLastQuery(pQuery)) { if (QUERY_IS_ASC_QUERY(pQuery)) { - dTrace(msg, GET_QINFO_ADDR(pQuery), "only-last stable", pQuery->order.order, TSQL_SO_DESC, + dTrace(msg, GET_QINFO_ADDR(pQuery), "only-last stable", pQuery->order.order, TSDB_ORDER_DESC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); } - pQuery->order.order = TSQL_SO_DESC; + pQuery->order.order = TSDB_ORDER_DESC; } } } @@ -2104,8 +2090,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI continue; } - int32_t colInBuf = pQuery->pSelectExpr[i].pBase.colInfo.colIdxInBuf; - + int32_t colInBuf = 0;//pQuery->pSelectExpr[i].pBase.colInfo.colIdxInBuf; SInterpInfo *pInterpInfo = (SInterpInfo *)pRuntimeEnv->pCtx[i].aOutputBuf; pInterpInfo->pInterpDetail = calloc(1, sizeof(SInterpInfoDetail)); @@ -2276,12 +2261,12 @@ int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { // set the ts-comp file traverse order if (param != NULL) { - int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSQL_SO_ASC : TSQL_SO_DESC; + int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; tsBufSetTraverseOrder(pRuntimeEnv->pTSBuf, order); } assert(0); - // int32_t ret = setupQueryRuntimeEnv(pMeter, pQuery, &pQInfo->runtimeEnv, pTagSchemaInfo, TSQL_SO_ASC, true); + // int32_t ret = setupQueryRuntimeEnv(pMeter, pQuery, &pQInfo->runtimeEnv, pTagSchemaInfo, TSDB_ORDER_ASC, true); // if (ret != TSDB_CODE_SUCCESS) { // return ret; // } @@ -2403,7 +2388,7 @@ static bool needToLoadDataBlock(SQuery *pQuery, SDataStatis *pDataStatis, SQLFun #if 0 for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - int32_t colIndex = pFilterInfo->info.colIdx; + int32_t colIndex = pFilterInfo->info.colIndex; // this column not valid in current data block if (colIndex < 0 || pDataStatis[colIndex].colId != pFilterInfo->info.data.colId) { @@ -2481,7 +2466,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl if (r == BLK_DATA_NO_NEEDED) { qTrace("QInfo:%p slot:%d, data block ignored, brange:%" PRId64 "-%" PRId64 ", rows:%d", - GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->size); + GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); } else if (r == BLK_DATA_FILEDS_NEEDED) { if (tsdbRetrieveDataBlockStatisInfo(pRuntimeEnv->pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { // return DISK_DATA_LOAD_FAILED; @@ -2501,7 +2486,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl * filter the data block according to the value filter condition. * no need to load the data block, continue for next block */ - if (!needToLoadDataBlock(pQuery, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->size)) { + if (!needToLoadDataBlock(pQuery, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { #if defined(_DEBUG_VIEW) dTrace("QInfo:%p fileId:%d, slot:%d, block discarded by per-filter", GET_QINFO_ADDR(pQuery), pQuery->fileId, pQuery->slot); @@ -2577,8 +2562,6 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - int64_t cnt = 0; - dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order); @@ -2586,43 +2569,66 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { while (tsdbNextDataBlock(pQueryHandle)) { // check if query is killed or not set the status of query to pass the status check if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { - return cnt; + return 0; } SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); + // todo extract methods if (isIntervalQuery(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == 0) { - TSKEY skey1, ekey1; - STimeWindow w = {0}; + TSKEY skey1, ekey1; + STimeWindow w = {0}; SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (QUERY_IS_ASC_QUERY(pQuery)) { - doGetAlignedIntervalQueryRangeImpl(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, + getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &skey1, &ekey1, &w); pWindowResInfo->startTime = w.skey; pWindowResInfo->prevSKey = w.skey; } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - TSKEY winStart = blockInfo.window.ekey - pQuery->intervalTime; - doGetAlignedIntervalQueryRangeImpl(pQuery, winStart, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1, - &w); + TSKEY start = blockInfo.window.ekey - pQuery->intervalTime; + getAlignQueryTimeWindow(pQuery, start, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1, &w); pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->prevSKey = w.skey; } } + + // in case of prj/diff query, ensure the output buffer is sufficient to accomodate the results of current block + if (!isIntervalQuery(pQuery) && !isGroupbyNormalCol(pQuery->pGroupbyExpr) && !isFixedOutputQuery(pQuery)) { + SResultRec* pRec = &pQuery->rec; + + if (pQuery->rec.capacity - pQuery->rec.rows < blockInfo.rows) { + int32_t remain = pRec->capacity - pRec->rows; + int32_t newSize = pRec->capacity + (blockInfo.rows - remain); + + for(int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t bytes = pQuery->pSelectExpr[i].resBytes; + + char* tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(SData)); + if (tmp == NULL) { // todo handle the oom + } else { + pQuery->sdata[i] = (SData*) tmp; + } + + // set the pCtx output buffer position + pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + pRec->rows*bytes; + } + + pRec->capacity = newSize; + } + } - int32_t numOfRes = 0; SDataStatis *pStatis = NULL; SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); - int32_t forwardStep = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, &numOfRes, + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, &pRuntimeEnv->windowResInfo, pDataBlock); - dTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d", - GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.size); + dTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", + GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes); // save last access position - cnt += forwardStep; if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { break; } @@ -2645,7 +2651,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { } } - return cnt; + return 0; } static void updatelastkey(SQuery *pQuery, STableQueryInfo *pTableQInfo) { pTableQInfo->lastKey = pQuery->lastKey; } @@ -2683,19 +2689,19 @@ void vnodeSetTagValueInParam(tSidSet *pSidSet, SQueryRuntimeEnv *pRuntimeEnv, vo } else { // set tag value, by which the results are aggregated. for (int32_t idx = 0; idx < pQuery->numOfOutputCols; ++idx) { - SColIndexEx *pColEx = &pQuery->pSelectExpr[idx].pBase.colInfo; + SColIndex *pColEx = &pQuery->pSelectExpr[idx].pBase.colInfo; // ts_comp column required the tag value for join filter if (!TSDB_COL_IS_TAG(pColEx->flag)) { continue; } - doSetTagValueInParam(pTagSchema, pColEx->colIdx, pMeterSidInfo, &pRuntimeEnv->pCtx[idx].tag); + doSetTagValueInParam(pTagSchema, pColEx->colIndex, pMeterSidInfo, &pRuntimeEnv->pCtx[idx].tag); } // set the join tag for first column SSqlFuncExprMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; - if (pFuncMsg->functionId == TSDB_FUNC_TS && pFuncMsg->colInfo.colIdx == PRIMARYKEY_TIMESTAMP_COL_INDEX && + if (pFuncMsg->functionId == TSDB_FUNC_TS && pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX && pRuntimeEnv->pTSBuf != NULL) { assert(pFuncMsg->numOfParams == 1); doSetTagValueInParam(pTagSchema, pFuncMsg->arg->argValue.i64, pMeterSidInfo, &pRuntimeEnv->pCtx[0].tag); @@ -2822,13 +2828,13 @@ void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOf for (int32_t i = 0; i < numOfCols; ++i) { switch (pQuery->pSelectExpr[i].resType) { case TSDB_DATA_TYPE_BINARY: { - int32_t colIdx = pQuery->pSelectExpr[i].pBase.colInfo.colIdx; + int32_t colIndex = pQuery->pSelectExpr[i].pBase.colInfo.colIndex; int32_t type = 0; if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[i].pBase.colInfo.flag)) { type = pQuery->pSelectExpr[i].resType; } else { - type = pMeterObj->schema[colIdx].type; + type = pMeterObj->schema[colIndex].type; } printBinaryData(pQuery->pSelectExpr[i].pBase.functionId, pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j, type); @@ -2979,9 +2985,9 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { offset += pData->numOfElems; } - assert(pQuery->rec.size == 0); + assert(pQuery->rec.rows == 0); - pQuery->rec.size += rows; + pQuery->rec.rows += rows; pQInfo->offset += 1; } @@ -3207,8 +3213,8 @@ static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo * for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; - if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSQL_SO_DESC) || - ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSQL_SO_ASC)) { + if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_DESC) || + ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_ASC)) { buf->resultInfo[j].complete = false; } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { buf->resultInfo[j].complete = true; @@ -3233,8 +3239,8 @@ void disableFunctForTableSuppleScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j]; - if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSQL_SO_DESC) || - ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSQL_SO_ASC)) { + if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_DESC) || + ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_ASC)) { pCtx->resultInfo->complete = false; } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { pCtx->resultInfo->complete = true; @@ -3310,7 +3316,7 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } - memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].resBytes * pQuery->rec.capacity); + memset(pQuery->sdata[i]->data, 0, (size_t) pQuery->pSelectExpr[i].resBytes * pQuery->rec.capacity); } initCtxOutputBuf(pRuntimeEnv); @@ -3349,22 +3355,22 @@ void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + pRuntimeEnv->pCtx[j].currentStage = 0; - aAggs[functionId].init(&pRuntimeEnv->pCtx[j]); } } void doSkipResults(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - if (pQuery->rec.size == 0 || pQuery->limit.offset == 0) { + if (pQuery->rec.rows == 0 || pQuery->limit.offset == 0) { return; } - if (pQuery->rec.size <= pQuery->limit.offset) { - pQuery->limit.offset -= pQuery->rec.size; + if (pQuery->rec.rows <= pQuery->limit.offset) { + pQuery->limit.offset -= pQuery->rec.rows; - pQuery->rec.size = 0; + pQuery->rec.rows = 0; // pQuery->pointsOffset = pQuery->rec.pointsToRead; // clear all data in result buffer resetCtxOutputBuf(pRuntimeEnv); @@ -3373,7 +3379,7 @@ void doSkipResults(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->status &= (~QUERY_RESBUF_FULL); } else { int32_t numOfSkip = (int32_t)pQuery->limit.offset; - pQuery->rec.size -= numOfSkip; + pQuery->rec.rows -= numOfSkip; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; @@ -3528,6 +3534,7 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { doScanAllDataBlocks(pRuntimeEnv); if (!needScanDataBlocksAgain(pRuntimeEnv)) { + // restore the status if (pRuntimeEnv->scanFlag == REPEAT_SCAN) { pQuery->status = status; @@ -3547,7 +3554,6 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { /* check if query is killed or not */ if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { -// setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } } @@ -3678,7 +3684,7 @@ void restoreIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *p /** * set output buffer for different group * @param pRuntimeEnv - * @param pDataBlockInfoEx + * @param pDataBlockInfo */ void setExecutionContext(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo, int32_t meterIdx, int32_t groupIdx, TSKEY nextKey) { @@ -3793,7 +3799,7 @@ void setIntervalQueryRange(STableQueryInfo *pTableQueryInfo, SQInfo *pQInfo, TSK STimeWindow w = {0}; SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo; - doGetAlignedIntervalQueryRangeImpl(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w); + getAlignQueryTimeWindow(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w); pWindowResInfo->startTime = pQuery->window.skey; // windowSKey may be 0 in case of 1970 timestamp if (pWindowResInfo->prevSKey == 0) { @@ -3863,7 +3869,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde dTrace("QInfo:%p start to copy data from windowResInfo to pQuery buf", GET_QINFO_ADDR(pQuery)); int32_t totalSubset = getNumOfSubset(pQInfo); - if (orderType == TSQL_SO_ASC) { + if (orderType == TSDB_ORDER_ASC) { startIdx = pQInfo->subgroupIdx; step = 1; } else { // desc order copy all data @@ -3929,12 +3935,12 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResult *result) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSQL_SO_ASC; + int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSDB_ORDER_ASC; int32_t numOfResult = doCopyToSData(pQInfo, result, orderType); - pQuery->rec.size += numOfResult; + pQuery->rec.rows += numOfResult; - assert(pQuery->rec.size <= pQuery->rec.capacity); + assert(pQuery->rec.rows <= pQuery->rec.capacity); } static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableDataInfo *pTableDataInfo) { @@ -3958,15 +3964,12 @@ void stableApplyFunctionsOnBlock(SQueryRuntimeEnv* pRuntimeEnv, STableDataInfo * STableQueryInfo * pTableQueryInfo = pTableDataInfo->pTableQInfo; SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; - int32_t numOfRes = 0; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { // numOfRes = rowwiseApplyAllFunctions(pRuntimeEnv, &forwardStep, pFields, pDataBlockInfo, pWindowResInfo); } else { - numOfRes = blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); } - assert(numOfRes >= 0); - updateWindowResNumOfRes(pRuntimeEnv, pTableDataInfo); updatelastkey(pQuery, pTableQueryInfo); } @@ -3982,7 +3985,7 @@ bool vnodeHasRemainResults(void *handle) { SQuery * pQuery = pRuntimeEnv->pQuery; SInterpolationInfo *pInterpoInfo = &pRuntimeEnv->interpoInfo; - if (pQuery->limit.limit > 0 && pQuery->rec.size >= pQuery->limit.limit) { + if (pQuery->limit.limit > 0 && pQuery->rec.rows >= pQuery->limit.limit) { return false; } @@ -4153,7 +4156,7 @@ void vnodePrintQueryStatistics(SQInfo *pQInfo) { #endif } -int32_t doInitializeQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) { +int32_t doInitQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -4182,13 +4185,15 @@ int32_t doInitializeQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTable pRuntimeEnv->pQuery = pQuery; pRuntimeEnv->pTSBuf = param; pRuntimeEnv->cur.vnodeIndex = -1; + pRuntimeEnv->stableQuery = isSTableQuery; + if (param != NULL) { - int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSQL_SO_ASC : TSQL_SO_DESC; + int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; tsBufSetTraverseOrder(pRuntimeEnv->pTSBuf, order); } // create runtime environment - code = setupQueryRuntimeEnv(pRuntimeEnv, NULL, pQuery->order.order, isSTableQuery); + code = setupQueryRuntimeEnv(pRuntimeEnv, NULL, pQuery->order.order); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4271,8 +4276,6 @@ int32_t doInitializeQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTable // the pQuery->window.skey is changed during normalizedFirstQueryRange, so set the newest lastkey value pQuery->lastKey = pQuery->window.skey; - pRuntimeEnv->stableQuery = false; - return TSDB_CODE_SUCCESS; } @@ -4282,7 +4285,7 @@ static UNUSED_FUNC bool isGroupbyEachTable(SSqlGroupbyExpr *pGroupbyExpr, tSidSe } for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { - SColIndexEx *pColIndex = &pGroupbyExpr->columnInfo[i]; + SColIndex *pColIndex = &pGroupbyExpr->columnInfo[i]; if (pColIndex->flag == TSDB_COL_TAG) { // assert(pSidset->numOfTables == pSidset->numOfSubSet); return true; @@ -4455,7 +4458,7 @@ static UNUSED_FUNC int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, i // accumulate the point interpolation result if (numOfRes > 0) { - pQuery->rec.size += numOfRes; + pQuery->rec.rows += numOfRes; forwardCtxOutputBuf(pRuntimeEnv, numOfRes); } @@ -4822,13 +4825,13 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); } - pQuery->rec.size += pQuery->rec.size; + pQuery->rec.rows += pQuery->rec.rows; - if (pQuery->rec.size == 0) { + if (pQuery->rec.rows == 0) { // vnodePrintQueryStatistics(pSupporter); } - dTrace("QInfo:%p current:%lldd, total:%lldd", pQInfo, pQuery->rec.size, pQuery->rec.total); + dTrace("QInfo:%p current:%lldd, total:%lldd", pQInfo, pQuery->rec.rows, pQuery->rec.total); return; } @@ -4885,7 +4888,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { } // handle the limitation of output buffer - dTrace("QInfo:%p points returned:%d, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); + dTrace("QInfo:%p points returned:%d, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); } /* @@ -4906,7 +4909,7 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { } // since the numOfOutputElems must be identical for all sql functions that are allowed to be executed simutanelously. - pQuery->rec.size = getNumOfResult(pRuntimeEnv); + pQuery->rec.rows = getNumOfResult(pRuntimeEnv); // assert(pQuery->size <= pQuery->pointsToRead && // Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED)); @@ -4918,12 +4921,12 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { doSkipResults(pRuntimeEnv); doRevisedResultsByLimit(pQInfo); - pQuery->rec.size = pQuery->rec.size; + pQuery->rec.rows = pQuery->rec.rows; } -static void tableMultiOutputProcessor(SQInfo *pQInfo) { +static void tableMultiOutputProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; + SQuery *pQuery = pRuntimeEnv->pQuery; // for ts_comp query, re-initialized is not allowed if (!isTSCompQuery(pQuery)) { @@ -4938,8 +4941,8 @@ static void tableMultiOutputProcessor(SQInfo *pQInfo) { return; } - pQuery->rec.size = getNumOfResult(pRuntimeEnv); - if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->rec.size > 0) { + pQuery->rec.rows = getNumOfResult(pRuntimeEnv); + if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->rec.rows > 0) { doSkipResults(pRuntimeEnv); } @@ -4947,7 +4950,7 @@ static void tableMultiOutputProcessor(SQInfo *pQInfo) { * 1. if pQuery->size == 0, pQuery->limit.offset >= 0, still need to check data * 2. if pQuery->size > 0, pQuery->limit.offset must be 0 */ - if (pQuery->rec.size > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { break; } @@ -4959,17 +4962,16 @@ static void tableMultiOutputProcessor(SQInfo *pQInfo) { doRevisedResultsByLimit(pQInfo); if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - dTrace("QInfo:%p query paused due to buffer limitation, next qrange:%" PRId64 "-%" PRId64, - pQInfo, pQuery->lastKey, pQuery->window.ekey); + dTrace("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, + pQInfo, pQuery->lastKey, pQuery->window.ekey); } // dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned, totalRead:%d totalReturn:%d", pQInfo, pMeterObj->vnode, // pMeterObj->sid, pMeterObj->meterId, pQuery->size, pQInfo->size, pQInfo->pointsReturned); -// pQuery->pointsOffset = pQuery->pointsToRead; //restore the available buffer -// if (!isTSCompQuery(pQuery)) { -// assert(pQuery->size <= pQuery->pointsToRead); -// } + if (!isTSCompQuery(pQuery)) { + assert(pQuery->rec.rows <= pQuery->rec.capacity); + } } static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { @@ -5016,7 +5018,7 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { if (isIntervalQuery(pQuery)) { pQInfo->subgroupIdx = 0; // always start from 0 - pQuery->rec.size = 0; + pQuery->rec.rows = 0; copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); @@ -5027,37 +5029,37 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { doRevisedResultsByLimit(pQInfo); break; } else { - taosInterpoSetStartInfo(&pRuntimeEnv->interpoInfo, pQuery->rec.size, pQuery->interpoType); + taosInterpoSetStartInfo(&pRuntimeEnv->interpoInfo, pQuery->rec.rows, pQuery->interpoType); SData **pInterpoBuf = pRuntimeEnv->pInterpoBuf; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - memcpy(pInterpoBuf[i]->data, pQuery->sdata[i]->data, pQuery->rec.size * pQuery->pSelectExpr[i].resBytes); + memcpy(pInterpoBuf[i]->data, pQuery->sdata[i]->data, pQuery->rec.rows * pQuery->pSelectExpr[i].resBytes); } numOfInterpo = 0; - pQuery->rec.size = vnodeQueryResultInterpolate( - pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, pQuery->rec.size, &numOfInterpo); + pQuery->rec.rows = vnodeQueryResultInterpolate( + pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, pQuery->rec.rows, &numOfInterpo); - dTrace("QInfo: %p interpo completed, final:%d", pQInfo, pQuery->rec.size); - if (pQuery->rec.size > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + dTrace("QInfo: %p interpo completed, final:%d", pQInfo, pQuery->rec.rows); + if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { doRevisedResultsByLimit(pQInfo); break; } // no result generated yet, continue retrieve data - pQuery->rec.size = 0; + pQuery->rec.rows = 0; } } // all data scanned, the group by normal column can return if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // todo refactor with merge interval time result pQInfo->subgroupIdx = 0; - pQuery->rec.size = 0; + pQuery->rec.rows = 0; copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); } - pQuery->rec.size += pQuery->rec.size; + pQuery->rec.rows += pQuery->rec.rows; pQInfo->pointsInterpo += numOfInterpo; // dTrace("%p vid:%d sid:%d id:%s, %d points returned %d points interpo, totalRead:%d totalInterpo:%d @@ -5066,7 +5068,7 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { // pQInfo->size - pQInfo->pointsInterpo, pQInfo->pointsInterpo, pQInfo->pointsReturned); } -static void singleTableQueryImpl(SQInfo* pQInfo) { +static void tableQueryImpl(SQInfo* pQInfo) { SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; SQuery* pQuery = pRuntimeEnv->pQuery; @@ -5077,15 +5079,15 @@ static void singleTableQueryImpl(SQInfo* pQInfo) { */ int32_t numOfInterpo = 0; int32_t remain = taosNumOfRemainPoints(&pRuntimeEnv->interpoInfo); - pQuery->rec.size = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, + pQuery->rec.rows = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo); doRevisedResultsByLimit(pQInfo); pQInfo->pointsInterpo += numOfInterpo; - pQuery->rec.size += pQuery->rec.size; + pQuery->rec.rows += pQuery->rec.rows; - dTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); + dTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); sem_post(&pQInfo->dataReady); return; } @@ -5097,17 +5099,17 @@ static void singleTableQueryImpl(SQInfo* pQInfo) { ((isIntervalQuery(pQuery) && pQuery->rec.total < pQuery->limit.limit))) { // todo limit the output for interval query? - pQuery->rec.size = 0; + pQuery->rec.rows = 0; pQInfo->subgroupIdx = 0; // always start from 0 if (pRuntimeEnv->windowResInfo.size > 0) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); - pQuery->rec.size += pQuery->rec.size; + pQuery->rec.rows += pQuery->rec.rows; clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); - if (pQuery->rec.size > 0) { - dTrace("QInfo:%p %d rows returned from group results, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); + if (pQuery->rec.rows > 0) { + dTrace("QInfo:%p %d rows returned from group results, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); sem_post(&pQInfo->dataReady); return; } @@ -5121,23 +5123,21 @@ static void singleTableQueryImpl(SQInfo* pQInfo) { } // number of points returned during this query - pQuery->rec.size = 0; + pQuery->rec.rows = 0; int64_t st = taosGetTimestampUs(); // group by normal column, sliding window query, interval query are handled by interval query processor if (isIntervalQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) tableIntervalProcessor(pQInfo); - } else { - if (isFixedOutputQuery(pQuery)) { - assert(pQuery->checkBufferInLoop == 0); - - tableFixedOutputProcessor(pQInfo); - } else { // diff/add/multiply/subtract/division - assert(pQuery->checkBufferInLoop == 1); - tableMultiOutputProcessor(pQInfo); - } + } else if (isFixedOutputQuery(pQuery)) { + assert(pQuery->checkBuffer == 0); + tableFixedOutputProcessor(pQInfo); + } else { // diff/add/multiply/subtract/division + assert(pQuery->checkBuffer == 1); + tableMultiOutputProcess(pQInfo); } + // record the total elapsed time pQInfo->elapsedTime += (taosGetTimestampUs() - st); @@ -5145,16 +5145,16 @@ static void singleTableQueryImpl(SQInfo* pQInfo) { if (isQueryKilled(pQInfo)) { dTrace("QInfo:%p query is killed", pQInfo); } else { - dTrace("QInfo:%p query task completed, %" PRId64 " rows will returned, total:%" PRId64 " rows", pQInfo, pQuery->rec.size, + dTrace("QInfo:%p query task completed, %" PRId64 " rows will returned, total:%" PRId64 " rows", pQInfo, pQuery->rec.rows, pQuery->rec.total); } sem_post(&pQInfo->dataReady); } -static void multiTableQueryImpl(SQInfo* pQInfo) { +static void stableQueryImpl(SQInfo* pQInfo) { SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - pQuery->rec.size = 0; + pQuery->rec.rows = 0; int64_t st = taosGetTimestampUs(); @@ -5162,7 +5162,7 @@ static void multiTableQueryImpl(SQInfo* pQInfo) { (isFixedOutputQuery(pQuery) && (!isPointInterpoQuery(pQuery)) && !isGroupbyNormalCol(pQuery->pGroupbyExpr))) { multiTableQueryProcess(pQInfo); } else { - assert((pQuery->checkBufferInLoop == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || + assert((pQuery->checkBuffer == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)); vnodeSTableSeqProcessor(pQInfo); @@ -5172,7 +5172,7 @@ static void multiTableQueryImpl(SQInfo* pQInfo) { pQInfo->elapsedTime += (taosGetTimestampUs() - st); // taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->size, pQInfo->query.interpoType); - if (pQuery->rec.size == 0) { + if (pQuery->rec.rows == 0) { int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); dTrace("QInfo:%p over, %d tables queried, %d points are returned", pQInfo, numOfTables, pQuery->rec.total); // vnodePrintQueryStatistics(pSupporter); @@ -5267,7 +5267,7 @@ static char* createTableIdList(SQueryTableMsg* pQueryMsg, char* pMsg, SArray** p * @return */ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr, - wchar_t** tagCond) { + char** tagCond) { pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); @@ -5285,7 +5285,6 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->numOfOutputCols = htons(pQueryMsg->numOfOutputCols); pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); - pQueryMsg->nameCondLen = htons(pQueryMsg->nameCondLen); pQueryMsg->tsOffset = htonl(pQueryMsg->tsOffset); pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); @@ -5345,7 +5344,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { (*pExpr)[i] = pExprMsg; - pExprMsg->colInfo.colIdx = htons(pExprMsg->colInfo.colIdx); + pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); pExprMsg->functionId = htons(pExprMsg->functionId); @@ -5399,7 +5398,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); pQueryMsg->orderType = htons(pQueryMsg->orderType); - pMsg += sizeof(SColIndexEx) * pQueryMsg->numOfGroupCols; + pMsg += sizeof(SColIndex) * pQueryMsg->numOfGroupCols; } else { pQueryMsg->groupbyTagIds = 0; } @@ -5418,8 +5417,8 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, // the tag query condition expression string is located at the end of query msg if (pQueryMsg->tagCondLen > 0) { - *tagCond = calloc(1, pQueryMsg->tagCondLen * TSDB_NCHAR_SIZE); - memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen * TSDB_NCHAR_SIZE); + *tagCond = calloc(1, pQueryMsg->tagCondLen); + memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); } dTrace("qmsg:%p query on %d table(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, ts order:%d, " @@ -5469,10 +5468,10 @@ static int32_t buildAirthmeticExprFromMsg(SSqlFunctionExpr *pExpr, SQueryTableMs // there may be duplicated referenced columns. num = i + 1; - pBinaryExprInfo->pReqColumns = malloc(sizeof(SColIndexEx) * num); + pBinaryExprInfo->pReqColumns = malloc(sizeof(SColIndex) * num); for (int32_t k = 0; k < num; ++k) { - SColIndexEx* pColIndex = &pBinaryExprInfo->pReqColumns[k]; + SColIndex* pColIndex = &pBinaryExprInfo->pReqColumns[k]; pColIndex->colId = ids[k]; } @@ -5569,19 +5568,19 @@ static SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, int3 // using group by tag columns SSqlGroupbyExpr *pGroupbyExpr = - (SSqlGroupbyExpr *)malloc(sizeof(SSqlGroupbyExpr) + pQueryMsg->numOfGroupCols * sizeof(SColIndexEx)); + (SSqlGroupbyExpr *)malloc(sizeof(SSqlGroupbyExpr) + pQueryMsg->numOfGroupCols * sizeof(SColIndex)); if (pGroupbyExpr == NULL) { *code = TSDB_CODE_SERV_OUT_OF_MEMORY; return NULL; } - SColIndexEx *pGroupbyColInfo = (SColIndexEx *)pQueryMsg->groupbyTagIds; + SColIndex *pGroupbyColInfo = (SColIndex *)pQueryMsg->groupbyTagIds; pGroupbyExpr->numOfGroupCols = pQueryMsg->numOfGroupCols; pGroupbyExpr->orderType = pQueryMsg->orderType; pGroupbyExpr->orderIndex = pQueryMsg->orderByIdx; - memcpy(pGroupbyExpr->columnInfo, pGroupbyColInfo, sizeof(SColIndexEx) * pGroupbyExpr->numOfGroupCols); + memcpy(pGroupbyExpr->columnInfo, pGroupbyColInfo, sizeof(SColIndex) * pGroupbyExpr->numOfGroupCols); return pGroupbyExpr; } @@ -5631,9 +5630,9 @@ static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { return TSDB_CODE_INVALID_QUERY_MSG; } - if ((lower == TSDB_RELATION_LARGE_EQUAL || lower == TSDB_RELATION_LARGE) && + if ((lower == TSDB_RELATION_GREATER_EQUAL || lower == TSDB_RELATION_GREATER) && (upper == TSDB_RELATION_LESS_EQUAL || upper == TSDB_RELATION_LESS)) { - if (lower == TSDB_RELATION_LARGE_EQUAL) { + if (lower == TSDB_RELATION_GREATER_EQUAL) { if (upper == TSDB_RELATION_LESS_EQUAL) { pSingleColFilter->fp = rangeFilterArray[4]; } else { @@ -5674,16 +5673,16 @@ static void doUpdateExprColumnIndex(SQuery* pQuery) { // int32_t i = 0, j = 0; // while (i < pQuery->numOfCols && j < pMeterObj->numOfColumns) { // if (pQuery->colList[i].data.colId == pMeterObj->schema[j].colId) { -// pQuery->colList[i++].colIdx = (int16_t)j++; +// pQuery->colList[i++].colIndex = (int16_t)j++; // } else if (pQuery->colList[i].data.colId < pMeterObj->schema[j].colId) { -// pQuery->colList[i++].colIdx = -1; +// pQuery->colList[i++].colIndex = -1; // } else if (pQuery->colList[i].data.colId > pMeterObj->schema[j].colId) { // j++; // } // } // while (i < pQuery->numOfCols) { -// pQuery->colList[i++].colIdx = -1; // not such column in current meter +// pQuery->colList[i++].colIndex = -1; // not such column in current meter // } for(int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { @@ -5692,10 +5691,10 @@ static void doUpdateExprColumnIndex(SQuery* pQuery) { continue; } - SColIndexEx* pColIndexEx = &pSqlExprMsg->colInfo; + SColIndex* pColIndexEx = &pSqlExprMsg->colInfo; for(int32_t f = 0; f < pQuery->numOfCols; ++f) { if (pColIndexEx->colId == pQuery->colList[f].info.colId) { - pColIndexEx->colIdx = f; + pColIndexEx->colIndex = f; break; } } @@ -5776,7 +5775,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou // set the output buffer capacity pQuery->rec.capacity = 4096; - pQuery->rec.threshold = 2; + pQuery->rec.threshold = 4000; for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { assert(pExprs[col].interResBytes >= pExprs[col].resBytes); @@ -5855,7 +5854,7 @@ static bool isValidQInfo(void *param) { } static void freeQInfo(SQInfo *pQInfo); -static int32_t initializeQInfo(SQueryTableMsg *pQueryMsg, void* tsdb, SQInfo *pQInfo, bool isSTable) { +static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void* tsdb, SQInfo *pQInfo, bool isSTable) { int32_t code = TSDB_CODE_SUCCESS; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -5880,7 +5879,7 @@ static int32_t initializeQInfo(SQueryTableMsg *pQueryMsg, void* tsdb, SQInfo *pQ } // filter the qualified - if ((code = doInitializeQInfo(pQInfo, pTSBuf, tsdb, isSTable)) != TSDB_CODE_SUCCESS) { + if ((code = doInitQInfo(pQInfo, pTSBuf, tsdb, isSTable)) != TSDB_CODE_SUCCESS) { goto _error; } @@ -6004,11 +6003,11 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { pQuery->sdata[0]->data, strerror(errno)); } } else { - doCopyQueryResultToMsg(pQInfo, pQuery->rec.size, data); + doCopyQueryResultToMsg(pQInfo, pQuery->rec.rows, data); } - pQuery->rec.total += pQuery->rec.size; - dTrace("QInfo:%p current:%d, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); + pQuery->rec.total += pQuery->rec.rows; + dTrace("QInfo:%p current:%d, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); return TSDB_CODE_SUCCESS; @@ -6020,9 +6019,9 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) int32_t code = TSDB_CODE_SUCCESS; + char* tagCond = NULL; SArray *pTableIdList = NULL; SSqlFuncExprMsg** pExprMsg = NULL; - wchar_t* tagCond = NULL; if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond)) != TSDB_CODE_SUCCESS) { return code; } @@ -6050,21 +6049,23 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) goto _query_over; } - // super table query - SArray* res = NULL; bool isSTableQuery = false; + SArray* res = NULL; if ((pQueryMsg->queryType & TSDB_QUERY_TYPE_STABLE_QUERY) != 0) { isSTableQuery = true; STableId* id = taosArrayGet(pTableIdList, 0); id->uid = -1; - res = tsdbQueryTableList(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen); + res = taosArrayInit(8, sizeof(STableId)); + + /*int32_t ret =*/ tsdbQueryTags(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, res); if (taosArrayGetSize(res) == 0) { // no qualified table in stable query in this vnode code = TSDB_CODE_SUCCESS; goto _query_over; } } else { + assert(taosArrayGetSize(pTableIdList) == 1); res = pTableIdList; } @@ -6073,7 +6074,7 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) code = TSDB_CODE_SERV_OUT_OF_MEMORY; } - code = initializeQInfo(pQueryMsg, tsdb, *pQInfo, isSTableQuery); + code = initQInfo(pQueryMsg, tsdb, *pQInfo, isSTableQuery); _query_over: if (code != TSDB_CODE_SUCCESS) { @@ -6116,11 +6117,10 @@ void qTableQuery(SQInfo *pQInfo) { dTrace("QInfo:%p query task is launched", pQInfo); - int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); - if (numOfTables == 1) { - singleTableQueryImpl(pQInfo); + if (pQInfo->runtimeEnv.stableQuery) { + stableQueryImpl(pQInfo); } else { - multiTableQueryImpl(pQInfo); + tableQueryImpl(pQInfo); } // vnodeDecRefCount(pQInfo); @@ -6138,7 +6138,7 @@ int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo) { } sem_wait(&pQInfo->dataReady); - dTrace("QInfo:%p retrieve result info, rowsize:%d, rows:%d, code:%d", pQInfo, pQuery->rowSize, pQuery->rec.size, + dTrace("QInfo:%p retrieve result info, rowsize:%d, rows:%d, code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows, pQInfo->code); return pQInfo->code; @@ -6167,12 +6167,12 @@ int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* c } SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - size_t size = getResultSize(pQInfo, &pQuery->rec.size); + size_t size = getResultSize(pQInfo, &pQuery->rec.rows); *contLen = size + sizeof(SRetrieveTableRsp); // todo handle failed to allocate memory *pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen); - (*pRsp)->numOfRows = htonl(pQuery->rec.size); + (*pRsp)->numOfRows = htonl(pQuery->rec.rows); int32_t code = pQInfo->code; if (code == TSDB_CODE_SUCCESS) { @@ -6183,7 +6183,7 @@ int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* c (*pRsp)->useconds = 0; } - if (pQuery->rec.size > 0 && code == TSDB_CODE_SUCCESS) { + if (pQuery->rec.rows > 0 && code == TSDB_CODE_SUCCESS) { code = doDumpQueryResult(pQInfo, (*pRsp)->data); } else { setQueryStatus(pQuery, QUERY_OVER); diff --git a/src/query/src/sql.c b/src/query/src/sql.c index f1dce997fe..d52e58395a 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -2471,13 +2471,13 @@ static void yy_reduce( yymsp[-1].minor.yy380 = yylhsminor.yy380; break; case 152: /* sortorder ::= ASC */ -{yymsp[0].minor.yy250 = TSQL_SO_ASC; } +{yymsp[0].minor.yy250 = TSDB_ORDER_ASC; } break; case 153: /* sortorder ::= DESC */ -{yymsp[0].minor.yy250 = TSQL_SO_DESC;} +{yymsp[0].minor.yy250 = TSDB_ORDER_DESC;} break; case 154: /* sortorder ::= */ -{yymsp[1].minor.yy250 = TSQL_SO_ASC;} +{yymsp[1].minor.yy250 = TSDB_ORDER_ASC;} break; case 157: /* grouplist ::= grouplist COMMA item */ { diff --git a/src/util/inc/tcompare.h b/src/util/inc/tcompare.h new file mode 100644 index 0000000000..3ac0d3d170 --- /dev/null +++ b/src/util/inc/tcompare.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TCOMPARE_H +#define TDENGINE_TCOMPARE_H + +#include "os.h" + +int32_t compareInt32Val(const void *pLeft, const void *pRight); + +int32_t compareInt64Val(const void *pLeft, const void *pRight); + +int32_t compareInt16Val(const void *pLeft, const void *pRight); + +int32_t compareInt8Val(const void *pLeft, const void *pRight); + +int32_t compareIntDoubleVal(const void *pLeft, const void *pRight); + +int32_t compareDoubleIntVal(const void *pLeft, const void *pRight); + +int32_t compareDoubleVal(const void *pLeft, const void *pRight); + +int32_t compareStrVal(const void *pLeft, const void *pRight); + +int32_t compareWStrVal(const void *pLeft, const void *pRight); + +__compar_fn_t getKeyComparFunc(int32_t keyType); + +__compar_fn_t getComparFunc(int32_t type, int32_t filterDataType); + +#endif // TDENGINE_TCOMPARE_H diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c new file mode 100644 index 0000000000..401db9279a --- /dev/null +++ b/src/util/src/tcompare.c @@ -0,0 +1,192 @@ +#include "taosdef.h" +#include "tcompare.h" + +int32_t compareInt32Val(const void *pLeft, const void *pRight) { + int32_t ret = GET_INT32_VAL(pLeft) - GET_INT32_VAL(pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +int32_t compareInt64Val(const void *pLeft, const void *pRight) { + int32_t ret = GET_INT64_VAL(pLeft) - GET_INT64_VAL(pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +int32_t compareInt16Val(const void *pLeft, const void *pRight) { + int32_t ret = GET_INT16_VAL(pLeft) - GET_INT16_VAL(pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +int32_t compareInt8Val(const void *pLeft, const void *pRight) { + int32_t ret = GET_INT8_VAL(pLeft) - GET_INT8_VAL(pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +int32_t compareIntDoubleVal(const void *pLeft, const void *pRight) { + // int64_t lhs = ((SSkipListKey *)pLeft)->i64Key; + // double rhs = ((SSkipListKey *)pRight)->dKey; + // if (fabs(lhs - rhs) < FLT_EPSILON) { + // return 0; + // } else { + // return (lhs > rhs) ? 1 : -1; + // } + return 0; +} + +int32_t compareDoubleIntVal(const void *pLeft, const void *pRight) { + // double lhs = ((SSkipListKey *)pLeft)->dKey; + // int64_t rhs = ((SSkipListKey *)pRight)->i64Key; + // if (fabs(lhs - rhs) < FLT_EPSILON) { + // return 0; + // } else { + // return (lhs > rhs) ? 1 : -1; + // } + return 0; +} + +int32_t compareDoubleVal(const void *pLeft, const void *pRight) { + double ret = GET_DOUBLE_VAL(pLeft) - GET_DOUBLE_VAL(pRight); + if (fabs(ret) < FLT_EPSILON) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +int32_t compareStrVal(const void *pLeft, const void *pRight) { + int32_t ret = strcmp(pLeft, pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +int32_t compareWStrVal(const void *pLeft, const void *pRight) { + // SSkipListKey *pL = (SSkipListKey *)pLeft; + // SSkipListKey *pR = (SSkipListKey *)pRight; + // + // if (pL->nLen == 0 && pR->nLen == 0) { + // return 0; + // } + // + // // handle only one-side bound compare situation, there is only lower bound or only upper bound + // if (pL->nLen == -1) { + // return 1; // no lower bound, lower bound is minimum, always return -1; + // } else if (pR->nLen == -1) { + // return -1; // no upper bound, upper bound is maximum situation, always return 1; + // } + // + // int32_t ret = wcscmp(((SSkipListKey *)pLeft)->wpz, ((SSkipListKey *)pRight)->wpz); + // + // if (ret == 0) { + // return 0; + // } else { + // return ret > 0 ? 1 : -1; + // } + return 0; +} + +__compar_fn_t getComparFunc(int32_t type, int32_t filterDataType) { + __compar_fn_t comparFn = NULL; + + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: { + if (filterDataType == TSDB_DATA_TYPE_BIGINT) { + comparFn = compareInt64Val; + break; + } + } + case TSDB_DATA_TYPE_BOOL: { + if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { + comparFn = compareInt32Val; + } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { + comparFn = compareIntDoubleVal; + } + break; + } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { +// if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { +// comparFn = compareDoubleIntVal; +// } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { +// comparFn = compareDoubleVal; +// } + if (filterDataType == TSDB_DATA_TYPE_DOUBLE) { + comparFn = compareDoubleVal; + } + break; + } + case TSDB_DATA_TYPE_BINARY: + comparFn = compareStrVal; + break; + case TSDB_DATA_TYPE_NCHAR: + comparFn = compareWStrVal; + break; + default: + comparFn = compareInt32Val; + break; + } + + return comparFn; +} + +__compar_fn_t getKeyComparFunc(int32_t keyType) { + __compar_fn_t comparFn = NULL; + + switch (keyType) { + case TSDB_DATA_TYPE_TINYINT: + comparFn = compareInt8Val; + break; + case TSDB_DATA_TYPE_SMALLINT: + comparFn = compareInt16Val; + break; + case TSDB_DATA_TYPE_INT: + comparFn = compareInt32Val; + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + comparFn = compareInt64Val; + break; + case TSDB_DATA_TYPE_BOOL: + comparFn = compareInt32Val; + break; + + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + comparFn = compareDoubleVal; + break; + + case TSDB_DATA_TYPE_BINARY: + comparFn = compareStrVal; + break; + + case TSDB_DATA_TYPE_NCHAR: + comparFn = compareWStrVal; + break; + + default: + comparFn = compareInt32Val; + break; + } + + return comparFn; +} diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h index b2673413ae..ff3d4666fc 100644 --- a/src/vnode/tsdb/inc/tsdb.h +++ b/src/vnode/tsdb/inc/tsdb.h @@ -208,7 +208,7 @@ typedef struct SBlockInfo { typedef struct SDataBlockInfo { STimeWindow window; - int32_t size; + int32_t rows; int32_t numOfCols; int64_t uid; int32_t sid; @@ -335,7 +335,7 @@ SArray *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle); * @param pTagCond. tag query condition * */ -SArray *tsdbQueryTableList(tsdb_repo_t* tsdb, int64_t uid, const wchar_t *pTagCond, size_t len); +int32_t tsdbQueryTags(tsdb_repo_t* tsdb, int64_t uid, const char *pTagCond, size_t len, SArray* list); /** * clean up the query handle diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 61f4995e43..0910355272 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -6,9 +6,10 @@ #include #include #include +#include #include #include -#include +#include #include // #include "taosdef.h" @@ -738,6 +739,7 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { STableId tableId = {.uid = pBlock->uid, .tid = pBlock->tid}; STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, tableId); if (pTable == NULL) { + dError("failed to get table for insert, uid:%" PRIu64 ", tid:%d", tableId.uid, tableId.tid); return TSDB_CODE_INVALID_TABLE_ID; } diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index 0c6fc61701..71a59a349d 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -93,6 +93,11 @@ void tsdbFreeEncode(void *cont) { if (cont != NULL) free(cont); } +static char* getTagIndexKey(const void* pData) { + STable* table = *(STable**) pData; + return getTupleKey(table->tagVal); +} + int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { STsdbMeta *pMeta = (STsdbMeta *)pHandle; @@ -101,8 +106,8 @@ int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { if (pTable->type == TSDB_SUPER_TABLE) { pTable->pIndex = - tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, 0, 0, getTupleKey); - } + tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, 0, 0, getTagIndexKey); + } tsdbAddTableToMeta(pMeta, pTable, false); @@ -218,7 +223,7 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { super->tagSchema = tdDupSchema(pCfg->tagSchema); super->tagVal = tdDataRowDup(pCfg->tagValues); super->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, - 0, 0, getTupleKey); // Allow duplicate key, no lock + 0, 0, getTagIndexKey); // Allow duplicate key, no lock if (super->pIndex == NULL) { tdFreeSchema(super->schema); @@ -403,8 +408,6 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { pNode->level = level; SSkipList* list = pSTable->pIndex; - - memcpy(SL_GET_NODE_KEY(list, pNode), dataRowTuple(pTable->tagVal), colBytes(s)); memcpy(SL_GET_NODE_DATA(pNode), &pTable, POINTER_BYTES); tSkipListPut(list, pNode); diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index a44174ae66..30181bed1d 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -19,13 +19,14 @@ #include "tutil.h" #include "../../../query/inc/qast.h" +#include "../../../query/inc/tlosertree.h" #include "../../../query/inc/tsqlfunction.h" #include "tsdb.h" #include "tsdbMain.h" #define EXTRA_BYTES 2 -#define PRIMARY_TSCOL_REQUIRED(c) (((SColumnInfoEx *)taosArrayGet(c, 0))->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) -#define QUERY_IS_ASC_QUERY(o) (o == TSQL_SO_ASC) +#define PRIMARY_TSCOL_REQUIRED(c) (((SColumnInfoEx*)taosArrayGet(c, 0))->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) +#define QUERY_IS_ASC_QUERY(o) (o == TSDB_ORDER_ASC) #define QH_GET_NUM_OF_COLS(handle) (taosArrayGetSize((handle)->pColumns)) enum { @@ -46,9 +47,9 @@ typedef struct SQueryFilePos { typedef struct SDataBlockLoadInfo { SFileGroup* fileGroup; - int32_t slot; - int32_t sid; - SArray *pLoadedCols; + int32_t slot; + int32_t sid; + SArray* pLoadedCols; } SDataBlockLoadInfo; typedef struct SLoadCompBlockInfo { @@ -57,52 +58,31 @@ typedef struct SLoadCompBlockInfo { int32_t fileListIndex; } SLoadCompBlockInfo; -typedef struct SQueryFilesInfo { - SArray *pFileInfo; - int32_t current; // the memory mapped header file, NOTE: only one header file can be mmap. - int32_t vnodeId; - - int32_t headerFd; // header file fd - int64_t headerFileSize; - int32_t dataFd; - int32_t lastFd; - - char headerFilePath[PATH_MAX]; // current opened header file name - char dataFilePath[PATH_MAX]; // current opened data file name - char lastFilePath[PATH_MAX]; // current opened last file path - char dbFilePathPrefix[PATH_MAX]; -} SQueryFilesInfo; - typedef struct STableCheckInfo { - STableId tableId; - TSKEY lastKey; - STable * pTableObj; - int64_t offsetInHeaderFile; -// int32_t numOfBlocks; - int32_t start; - bool checkFirstFileBlock; - - SCompIdx* compIndex; - SCompInfo *pCompInfo; - - SDataCols* pDataCols; - SFileGroup* pFileGroup; - - SFileGroupIter fileIter; + STableId tableId; + TSKEY lastKey; + STable* pTableObj; + int64_t offsetInHeaderFile; + int32_t start; + bool checkFirstFileBlock; + SCompInfo* pCompInfo; + int32_t numOfBlocks; // number of qualified data blocks not the original blocks + + SDataCols* pDataCols; SSkipListIterator* iter; } STableCheckInfo; typedef struct { - SCompBlock *compBlock; - SField * fields; + SCompBlock* compBlock; + SField* fields; } SCompBlockFields; -typedef struct STableDataBlockInfoEx { +typedef struct STableBlockInfo { SCompBlockFields pBlock; - STableCheckInfo* pMeterDataInfo; + STableCheckInfo* pTableCheckInfo; int32_t blockIndex; - int32_t groupIdx; /* number of group is less than the total number of meters */ -} STableDataBlockInfoEx; + int32_t groupIdx; /* number of group is less than the total number of tables */ +} STableBlockInfo; enum { SINGLE_TABLE_MODEL = 1, @@ -110,17 +90,13 @@ enum { }; typedef struct STsdbQueryHandle { - STsdbRepo* pTsdb; - int8_t model; // access model, single table model or multi-table model + STsdbRepo* pTsdb; + int8_t model; // access model, single table model or multi-table model SQueryFilePos cur; // current position SQueryFilePos start; // the start position, used for secondary/third iteration - int32_t unzipBufSize; - char *unzipBuffer; - char *secondaryUnzipBuffer; SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */ - SQueryFilesInfo vnodeFileInfo; int16_t numOfRowsPerPage; uint16_t flag; // denotes reversed scan of data or not @@ -129,24 +105,29 @@ typedef struct STsdbQueryHandle { int32_t blockBufferSize; SCompBlock* pBlock; int32_t numOfBlocks; - SField ** pFields; - SArray * pColumns; // column list, SColumnInfoEx array list + SField** pFields; + SArray* pColumns; // column list, SColumnInfoEx array list bool locateStart; int32_t realNumOfRows; bool loadDataAfterSeek; // load data after seek. SArray* pTableCheckInfo; int32_t activeIndex; - - int32_t tableIndex; - bool isFirstSlot; - void * qinfo; // query info handle, for debug purpose - - STableDataBlockInfoEx *pDataBlockInfoEx; + + bool checkFiles; // check file stage + int32_t tableIndex; + bool isFirstSlot; + void* qinfo; // query info handle, for debug purpose + + STableBlockInfo* pDataBlockInfo; + + SFileGroup* pFileGroup; + SFileGroupIter fileIter; + SCompIdx* compIndex; } STsdbQueryHandle; -int32_t doAllocateBuf(STsdbQueryHandle *pQueryHandle, int32_t rowsPerFileBlock) { +int32_t doAllocateBuf(STsdbQueryHandle* pQueryHandle, int32_t rowsPerFileBlock) { // record the maximum column width among columns of this meter/metric - SColumnInfoEx *pColumn = taosArrayGet(pQueryHandle->pColumns, 0); + SColumnInfoEx* pColumn = taosArrayGet(pQueryHandle->pColumns, 0); int32_t maxColWidth = pColumn->info.bytes; for (int32_t i = 1; i < QH_GET_NUM_OF_COLS(pQueryHandle); ++i) { @@ -156,89 +137,65 @@ int32_t doAllocateBuf(STsdbQueryHandle *pQueryHandle, int32_t rowsPerFileBlock) } } - // only one unzip buffer required, since we can unzip each column one by one - pQueryHandle->unzipBufSize = (size_t)(maxColWidth * rowsPerFileBlock + EXTRA_BYTES); // plus extra_bytes - pQueryHandle->unzipBuffer = (char *)calloc(1, pQueryHandle->unzipBufSize); - - pQueryHandle->secondaryUnzipBuffer = (char *)calloc(1, pQueryHandle->unzipBufSize); - - if (pQueryHandle->unzipBuffer == NULL || pQueryHandle->secondaryUnzipBuffer == NULL) { - goto _error_clean; - } - return TSDB_CODE_SUCCESS; - -_error_clean: - tfree(pQueryHandle->unzipBuffer); - tfree(pQueryHandle->secondaryUnzipBuffer); - - return TSDB_CODE_SERV_OUT_OF_MEMORY; } -static void initQueryFileInfoFD(SQueryFilesInfo *pVnodeFilesInfo) { - pVnodeFilesInfo->current = -1; - pVnodeFilesInfo->headerFileSize = -1; - - pVnodeFilesInfo->headerFd = FD_INITIALIZER; // set the initial value - pVnodeFilesInfo->dataFd = FD_INITIALIZER; - pVnodeFilesInfo->lastFd = FD_INITIALIZER; -} - -static void vnodeInitDataBlockLoadInfo(SDataBlockLoadInfo *pBlockLoadInfo) { +static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; pBlockLoadInfo->sid = -1; pBlockLoadInfo->fileGroup = NULL; } -static void vnodeInitCompBlockLoadInfo(SLoadCompBlockInfo *pCompBlockLoadInfo) { +static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) { pCompBlockLoadInfo->sid = -1; pCompBlockLoadInfo->fileId = -1; pCompBlockLoadInfo->fileListIndex = -1; } -tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond, SArray *idList, SArray *pColumnInfo) { +tsdb_query_handle_t* tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond* pCond, SArray* idList, SArray* pColumnInfo) { // todo 1. filter not exist table // todo 2. add the reference count for each table that is involved in query - STsdbQueryHandle *pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); + STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); pQueryHandle->order = pCond->order; pQueryHandle->window = pCond->twindow; pQueryHandle->pTsdb = tsdb; + pQueryHandle->compIndex = calloc(10000, sizeof(SCompIdx)), pQueryHandle->loadDataAfterSeek = false; pQueryHandle->isFirstSlot = true; - + size_t size = taosArrayGetSize(idList); assert(size >= 1); pQueryHandle->pTableCheckInfo = taosArrayInit(size, sizeof(STableCheckInfo)); - for(int32_t i = 0; i < size; ++i) { - STableId id = *(STableId*) taosArrayGet(idList, i); - + for (int32_t i = 0; i < size; ++i) { + STableId id = *(STableId*)taosArrayGet(idList, i); + STableCheckInfo info = { - .lastKey = pQueryHandle->window.skey, - .tableId = id, - .pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id.uid), //todo this may be failed - .compIndex = calloc(10000, sizeof(SCompIdx)), - .pCompInfo = calloc(1, 1024), + .lastKey = pQueryHandle->window.skey, + .tableId = id, + .pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id.uid), // todo this may be failed + .pCompInfo = calloc(1, 1024), }; - + assert(info.pTableObj != NULL); taosArrayPush(pQueryHandle->pTableCheckInfo, &info); } - - pQueryHandle->model = (size > 1)? MULTI_TABLE_MODEL:SINGLE_TABLE_MODEL; - + + pQueryHandle->model = (size > 1) ? MULTI_TABLE_MODEL : SINGLE_TABLE_MODEL; + pQueryHandle->checkFiles = 1; + pQueryHandle->activeIndex = 0; - + // malloc buffer in order to load data from file int32_t numOfCols = taosArrayGetSize(pColumnInfo); size_t bufferCapacity = 4096; - + pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoEx)); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoEx *pCol = taosArrayGet(pColumnInfo, i); + SColumnInfoEx* pCol = taosArrayGet(pColumnInfo, i); SColumnInfoEx pDest = {{0}, 0}; pDest.pData = calloc(1, EXTRA_BYTES + bufferCapacity * pCol->info.bytes); @@ -250,152 +207,224 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond return NULL; } - initQueryFileInfoFD(&pQueryHandle->vnodeFileInfo); - vnodeInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); - vnodeInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); return (tsdb_query_handle_t)pQueryHandle; } -static bool hasMoreDataInCacheForSingleModel(STsdbQueryHandle* pHandle) { +static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); - + STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - - STable *pTable = pTableCheckInfo->pTableObj; + + STable* pTable = pTableCheckInfo->pTableObj; assert(pTable != NULL); - + // no data in cache, abort if (pTable->mem == NULL && pTable->imem == NULL) { return false; } - + // all data in mem are checked already. if (pTableCheckInfo->lastKey > pTable->mem->keyLast) { return false; } - + return true; } // todo dynamic get the daysperfile static int32_t getFileIdFromKey(TSKEY key) { - return (int32_t)(key / 10); // set the starting fileId + return (int32_t)(key / 10); // set the starting fileId } -static int32_t getFileCompInfo(STableCheckInfo* pCheckInfo, SFileGroup* fileGroup) { - // check open file failed +static int32_t binarySearchForBlockImpl(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order); + +static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks, int32_t type) { + // todo check open file failed + SFileGroup* fileGroup = pQueryHandle->pFileGroup; if (fileGroup->files[TSDB_FILE_TYPE_HEAD].fd == FD_INITIALIZER) { fileGroup->files[TSDB_FILE_TYPE_HEAD].fd = open(fileGroup->files[TSDB_FILE_TYPE_HEAD].fname, O_RDONLY); } - - tsdbLoadCompIdx(fileGroup, pCheckInfo->compIndex, 10000); // todo set dynamic max tables - SCompIdx* compIndex = &pCheckInfo->compIndex[pCheckInfo->tableId.tid]; - - if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file - - } else { - tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo); + + // load all the comp offset value for all tables in this file + tsdbLoadCompIdx(fileGroup, pQueryHandle->compIndex, 10000); // todo set dynamic max tables + + *numOfBlocks = 0; + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + + for (int32_t i = 0; i < numOfTables; ++i) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + + SCompIdx* compIndex = &pQueryHandle->compIndex[pCheckInfo->tableId.tid]; + if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file + + } else { + tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo); + + int32_t index = 0; + // discard the unqualified data block based on the query time window + int32_t start = binarySearchForBlockImpl(pCheckInfo->pCompInfo->blocks, compIndex->numOfSuperBlocks, + pQueryHandle->order, pCheckInfo->lastKey); + + if (type == QUERY_RANGE_GREATER_EQUAL) { + if (pCheckInfo->lastKey <= pCheckInfo->pCompInfo->blocks[start].keyLast) { + // break; + } else { + index = -1; + } + } else { + if (pCheckInfo->lastKey >= pCheckInfo->pCompInfo->blocks[start].keyFirst) { + // break; + } else { + index = -1; + } + } + + // not found in data blocks in current file + if (index == -1) { + continue; + } + + // todo speedup the procedure of locating end block + int32_t e = start; + while (e < compIndex->numOfSuperBlocks && + (pCheckInfo->pCompInfo->blocks[e].keyFirst <= pQueryHandle->window.ekey)) { + e += 1; + } + + if (start > 0) { + memmove(pCheckInfo->pCompInfo->blocks, &pCheckInfo->pCompInfo->blocks[start], (e - start) * sizeof(SCompBlock)); + } + + pCheckInfo->numOfBlocks = (e - start); + (*numOfBlocks) += pCheckInfo->numOfBlocks; + } } - + return TSDB_CODE_SUCCESS; } -static int32_t binarySearchForBlockImpl(SCompBlock *pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { +int32_t binarySearchForBlockImpl(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { int32_t firstSlot = 0; int32_t lastSlot = numOfBlocks - 1; - + int32_t midSlot = firstSlot; - + while (1) { numOfBlocks = lastSlot - firstSlot + 1; midSlot = (firstSlot + (numOfBlocks >> 1)); - + if (numOfBlocks == 1) break; - + if (skey > pBlock[midSlot].keyLast) { if (numOfBlocks == 2) break; - if ((order == TSQL_SO_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break; + if ((order == TSDB_ORDER_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break; firstSlot = midSlot + 1; } else if (skey < pBlock[midSlot].keyFirst) { - if ((order == TSQL_SO_ASC) && (skey > pBlock[midSlot - 1].keyLast)) break; + if ((order == TSDB_ORDER_ASC) && (skey > pBlock[midSlot - 1].keyLast)) break; lastSlot = midSlot - 1; } else { break; // got the slot } } - + return midSlot; } -static SDataBlockInfo getTrueDataBlockInfo(STsdbQueryHandle* pHandle, STableCheckInfo* pCheckInfo) { - SCompBlock *pDiskBlock = &pCheckInfo->pCompInfo->blocks[pHandle->cur.slot]; - +static SDataBlockInfo getTrueDataBlockInfo(STableCheckInfo* pCheckInfo, SCompBlock* pBlock) { SDataBlockInfo info = { - .window = {.skey = pDiskBlock->keyFirst, .ekey = pDiskBlock->keyLast}, - .numOfCols = pDiskBlock->numOfCols, - .size = pDiskBlock->numOfPoints, - .sid = pCheckInfo->tableId.tid, - .uid = pCheckInfo->tableId.uid, + .window = {.skey = pBlock->keyFirst, .ekey = pBlock->keyLast}, + .numOfCols = pBlock->numOfCols, + .rows = pBlock->numOfPoints, + .sid = pCheckInfo->tableId.tid, + .uid = pCheckInfo->tableId.uid, }; - + return info; } -SArray *getDefaultLoadColumns(STsdbQueryHandle *pQueryHandle, bool loadTS); -static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SDataCols* pCols, SArray *sa); +SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS); +static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock, + SArray* sa); +static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); + +static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) { + SCompData* data = calloc(1, sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols); -static bool doLoadDataFromFileBlock(STsdbQueryHandle *pQueryHandle) { - SQueryFilePos *cur = &pQueryHandle->cur; - - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; - - SCompData* data = calloc(1, sizeof(SCompData)+ sizeof(SCompCol)*pBlock->numOfCols); - data->numOfCols = pBlock->numOfCols; data->uid = pCheckInfo->pTableObj->tableId.uid; - - pCheckInfo->pDataCols = tdNewDataCols(1000, 2, 4096); - tdInitDataCols(pCheckInfo->pDataCols, pCheckInfo->pTableObj->schema); - - SFile* pFile = &pCheckInfo->pFileGroup->files[TSDB_FILE_TYPE_DATA]; + + bool blockLoaded = false; + SArray* sa = getDefaultLoadColumns(pQueryHandle, true); + + if (pCheckInfo->pDataCols == NULL) { + pCheckInfo->pDataCols = tdNewDataCols(1000, 2, 4096); + } + + tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj)); + + SFile* pFile = &pQueryHandle->pFileGroup->files[TSDB_FILE_TYPE_DATA]; if (pFile->fd == FD_INITIALIZER) { pFile->fd = open(pFile->fname, O_RDONLY); } - - tsdbLoadDataBlock(pFile, pBlock, 1, pCheckInfo->pDataCols, data); - return true; + + if (tsdbLoadDataBlock(pFile, pBlock, 1, pCheckInfo->pDataCols, data) == 0) { + SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; + + pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup; + pBlockLoadInfo->slot = pQueryHandle->cur.slot; + pBlockLoadInfo->sid = pCheckInfo->pTableObj->tableId.tid; + + blockLoaded = true; + } + + taosArrayDestroy(sa); + tfree(data); + + TSKEY* d = (TSKEY*)pCheckInfo->pDataCols->cols[PRIMARYKEY_TIMESTAMP_COL_INDEX].pData; + assert(d[0] == pBlock->keyFirst && d[pBlock->numOfPoints - 1] == pBlock->keyLast); + + return blockLoaded; } -static bool loadQualifiedDataFromFileBlock(STsdbQueryHandle *pQueryHandle) { - SQueryFilePos *cur = &pQueryHandle->cur; - - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; - - SArray *sa = getDefaultLoadColumns(pQueryHandle, true); +static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) { + SArray* sa = getDefaultLoadColumns(pQueryHandle, true); + SQueryFilePos* cur = &pQueryHandle->cur; + if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { // query ended in current block if (pQueryHandle->window.ekey < pBlock->keyLast) { - doLoadDataFromFileBlock(pQueryHandle); - filterDataInDataBlock(pQueryHandle, pCheckInfo->pDataCols, sa); - } else { // the whole block is loaded in to buffer + if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { + return false; + } + + SDataCols* pDataCols = pCheckInfo->pDataCols; + + if (pCheckInfo->lastKey > pBlock->keyFirst) { + cur->pos = + binarySearchForKey(pDataCols->cols[0].pData, pBlock->numOfPoints, pCheckInfo->lastKey, pQueryHandle->order); + } else { + cur->pos = 0; + } + + filterDataInDataBlock(pQueryHandle, pCheckInfo, pBlock, sa); + } else { // the whole block is loaded in to buffer pQueryHandle->realNumOfRows = pBlock->numOfPoints; } - } else {// todo desc query + } else { // todo desc query if (pQueryHandle->window.ekey > pBlock->keyFirst) { -// } } - + taosArrayDestroy(sa); return pQueryHandle->realNumOfRows > 0; } -bool moveToNextBlock(STsdbQueryHandle *pQueryHandle, int32_t step) { - SQueryFilePos *cur = &pQueryHandle->cur; - +bool moveToNextBlock(STsdbQueryHandle* pQueryHandle, int32_t step) { + SQueryFilePos* cur = &pQueryHandle->cur; + if (pQueryHandle->cur.fid >= 0) { /* * 1. ascending order. The last data block of data file @@ -405,74 +434,70 @@ bool moveToNextBlock(STsdbQueryHandle *pQueryHandle, int32_t step) { int32_t tid = pCheckInfo->tableId.tid; if ((step == QUERY_ASC_FORWARD_STEP && - (pQueryHandle->cur.slot == pCheckInfo->compIndex[tid].numOfSuperBlocks - 1)) || + (pQueryHandle->cur.slot == pQueryHandle->compIndex[tid].numOfSuperBlocks - 1)) || (step == QUERY_DESC_FORWARD_STEP && (pQueryHandle->cur.slot == 0))) { // temporarily keep the position value, in case of no data qualified when move forwards(backwards) -// SQueryFilePos save = pQueryHandle->cur; - SFileGroup* fgroup = tsdbGetFileGroupNext(&pCheckInfo->fileIter); + // SQueryFilePos save = pQueryHandle->cur; + pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); int32_t fid = -1; - if (fgroup != NULL) { - if ((fid = getFileCompInfo(pCheckInfo, fgroup)) < 0) { + int32_t numOfBlocks = 0; + + if (pQueryHandle->pFileGroup != NULL) { + if ((fid = getFileCompInfo(pQueryHandle, &numOfBlocks, 1)) < 0) { } else { cur->slot = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->numOfBlocks - 1; cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->pBlock[cur->slot].numOfPoints - 1; SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; - SCompData* data = calloc(1, sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols); - - data->numOfCols = pBlock->numOfCols; - data->uid = pCheckInfo->pTableObj->tableId.uid; - - cur->fid = fgroup->fileId; + cur->fid = pQueryHandle->pFileGroup->fileId; assert(cur->pos >= 0 && cur->fid >= 0 && cur->slot >= 0); if (pBlock->keyFirst > pQueryHandle->window.ekey) { // done return false; } - - loadQualifiedDataFromFileBlock(pQueryHandle); - return true; + + return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); } } else { // check data in cache pQueryHandle->cur.fid = -1; - return hasMoreDataInCacheForSingleModel(pQueryHandle); + return hasMoreDataInCache(pQueryHandle); } - } else { // next block in the same file + } else { // next block in the same file cur->slot += step; SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pBlock->numOfPoints - 1; - return loadQualifiedDataFromFileBlock(pQueryHandle); + return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); } } else { // data in cache - return hasMoreDataInCacheForSingleModel(pQueryHandle); + return hasMoreDataInCache(pQueryHandle); } - + return false; } -int vnodeBinarySearchKey(char *pValue, int num, TSKEY key, int order) { +int vnodeBinarySearchKey(char* pValue, int num, TSKEY key, int order) { int firstPos, lastPos, midPos = -1; int numOfPoints; - TSKEY *keyList; - + TSKEY* keyList; + if (num <= 0) return -1; - - keyList = (TSKEY *)pValue; + + keyList = (TSKEY*)pValue; firstPos = 0; lastPos = num - 1; - + if (order == 0) { // find the first position which is smaller than the key while (1) { if (key >= keyList[lastPos]) return lastPos; if (key == keyList[firstPos]) return firstPos; if (key < keyList[firstPos]) return firstPos - 1; - + numOfPoints = lastPos - firstPos + 1; midPos = (numOfPoints >> 1) + firstPos; - + if (key < keyList[midPos]) { lastPos = midPos - 1; } else if (key > keyList[midPos]) { @@ -481,13 +506,13 @@ int vnodeBinarySearchKey(char *pValue, int num, TSKEY key, int order) { break; } } - + } else { // find the first position which is bigger than the key while (1) { if (key <= keyList[firstPos]) return firstPos; if (key == keyList[lastPos]) return lastPos; - + if (key > keyList[lastPos]) { lastPos = lastPos + 1; if (lastPos >= num) @@ -495,10 +520,10 @@ int vnodeBinarySearchKey(char *pValue, int num, TSKEY key, int order) { else return lastPos; } - + numOfPoints = lastPos - firstPos + 1; midPos = (numOfPoints >> 1) + firstPos; - + if (key < keyList[midPos]) { lastPos = midPos - 1; } else if (key > keyList[midPos]) { @@ -508,20 +533,22 @@ int vnodeBinarySearchKey(char *pValue, int num, TSKEY key, int order) { } } } - + return midPos; } // only return the qualified data to client in terms of query time window, data rows in the same block but do not // be included in the query time window will be discarded -static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SDataCols* pCols, SArray *sa) { - SQueryFilePos *cur = &pQueryHandle->cur; - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - SDataBlockInfo blockInfo = getTrueDataBlockInfo(pQueryHandle, pCheckInfo); - +static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock, + SArray* sa) { + SQueryFilePos* cur = &pQueryHandle->cur; + SDataBlockInfo blockInfo = getTrueDataBlockInfo(pCheckInfo, pBlock); + + SDataCols* pCols = pCheckInfo->pDataCols; + int32_t endPos = cur->pos; if (QUERY_IS_ASC_QUERY(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { - endPos = blockInfo.size - 1; + endPos = blockInfo.rows - 1; pQueryHandle->realNumOfRows = endPos - cur->pos + 1; pCheckInfo->lastKey = blockInfo.window.ekey + 1; } else if (!QUERY_IS_ASC_QUERY(pQueryHandle->order) && pQueryHandle->window.ekey < blockInfo.window.skey) { @@ -529,8 +556,9 @@ static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SDataCols* pCo pQueryHandle->realNumOfRows = cur->pos + 1; pCheckInfo->lastKey = blockInfo.window.ekey - 1; } else { - endPos = vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfPoints, pQueryHandle->window.ekey, pQueryHandle->order); - + endPos = + vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfPoints, pQueryHandle->window.ekey, pQueryHandle->order); + if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { if (endPos < cur->pos) { pQueryHandle->realNumOfRows = 0; @@ -538,7 +566,7 @@ static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SDataCols* pCo } else { pQueryHandle->realNumOfRows = endPos - cur->pos; } - + pCheckInfo->lastKey = ((int64_t*)(pCols->cols[0].pData))[endPos] + 1; } else { if (endPos > cur->pos) { @@ -547,83 +575,84 @@ static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SDataCols* pCo } else { pQueryHandle->realNumOfRows = cur->pos - endPos; } - + assert(0); } } - + int32_t start = MIN(cur->pos, endPos); - + // move the data block in the front to data block if needed int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); - + for (int32_t i = 0; i < taosArrayGetSize(sa); ++i) { - int16_t colId = *(int16_t *)taosArrayGet(sa, i); - + int16_t colId = *(int16_t*)taosArrayGet(sa, i); + for (int32_t j = 0; j < numOfCols; ++j) { - SColumnInfoEx *pCol = taosArrayGet(pQueryHandle->pColumns, j); - + SColumnInfoEx* pCol = taosArrayGet(pQueryHandle->pColumns, j); + if (pCol->info.colId == colId) { SDataCol* pDataCol = &pCols->cols[i]; - memmove(pCol->pData, pDataCol->pData + pCol->info.bytes * start, pQueryHandle->realNumOfRows * pCol->info.bytes); + memmove(pCol->pData, pDataCol->pData + pCol->info.bytes * start, + pQueryHandle->realNumOfRows * pCol->info.bytes); break; } } } - - assert(pQueryHandle->realNumOfRows <= blockInfo.size); - + + assert(pQueryHandle->realNumOfRows <= blockInfo.rows); + // forward(backward) the position for cursor cur->pos = endPos; } -static SArray *getColumnIdList(STsdbQueryHandle *pQueryHandle) { +static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) { int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); - SArray *pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); + SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoEx *pCol = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoEx* pCol = taosArrayGet(pQueryHandle->pColumns, i); taosArrayPush(pIdList, &pCol->info.colId); } - + return pIdList; } -SArray *getDefaultLoadColumns(STsdbQueryHandle *pQueryHandle, bool loadTS) { - SArray *pLocalIdList = getColumnIdList(pQueryHandle); - +SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS) { + SArray* pLocalIdList = getColumnIdList(pQueryHandle); + // check if the primary time stamp column needs to load - int16_t colId = *(int16_t *)taosArrayGet(pLocalIdList, 0); - + int16_t colId = *(int16_t*)taosArrayGet(pLocalIdList, 0); + // the primary timestamp column does not be included in the the specified load column list, add it if (loadTS && colId != 0) { int16_t columnId = 0; taosArrayInsert(pLocalIdList, 0, &columnId); } - + return pLocalIdList; } -static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { +int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { int firstPos, lastPos, midPos = -1; int numOfPoints; - TSKEY *keyList; - + TSKEY* keyList; + if (num <= 0) return -1; - - keyList = (TSKEY *)pValue; + + keyList = (TSKEY*)pValue; firstPos = 0; lastPos = num - 1; - + if (order == 0) { // find the first position which is smaller than the key while (1) { if (key >= keyList[lastPos]) return lastPos; if (key == keyList[firstPos]) return firstPos; if (key < keyList[firstPos]) return firstPos - 1; - + numOfPoints = lastPos - firstPos + 1; midPos = (numOfPoints >> 1) + firstPos; - + if (key < keyList[midPos]) { lastPos = midPos - 1; } else if (key > keyList[midPos]) { @@ -632,13 +661,13 @@ static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { break; } } - + } else { // find the first position which is bigger than the key while (1) { if (key <= keyList[firstPos]) return firstPos; if (key == keyList[lastPos]) return lastPos; - + if (key > keyList[lastPos]) { lastPos = lastPos + 1; if (lastPos >= num) @@ -646,10 +675,10 @@ static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { else return lastPos; } - + numOfPoints = lastPos - firstPos + 1; midPos = (numOfPoints >> 1) + firstPos; - + if (key < keyList[midPos]) { lastPos = midPos - 1; } else if (key > keyList[midPos]) { @@ -659,513 +688,652 @@ static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { } } } - + return midPos; } -static bool getQualifiedDataBlock(STsdbQueryHandle *pQueryHandle, STableCheckInfo* pCheckInfo, int32_t type) { +static bool getQualifiedDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t type) { STsdbFileH* pFileHandle = tsdbGetFile(pQueryHandle->pTsdb); - int32_t fid = getFileIdFromKey(pCheckInfo->lastKey); - - tsdbInitFileGroupIter(pFileHandle, &pCheckInfo->fileIter, TSDB_FGROUP_ITER_FORWARD); - tsdbSeekFileGroupIter(&pCheckInfo->fileIter, fid); - pCheckInfo->pFileGroup = tsdbGetFileGroupNext(&pCheckInfo->fileIter); - + int32_t fid = getFileIdFromKey(pCheckInfo->lastKey); + + tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, TSDB_FGROUP_ITER_FORWARD); + tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); + pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); + SQueryFilePos* cur = &pQueryHandle->cur; - - TSKEY key = pCheckInfo->lastKey; - int32_t index = -1; int32_t tid = pCheckInfo->tableId.tid; - - while (pCheckInfo->pFileGroup != NULL) { - if (getFileCompInfo(pCheckInfo, pCheckInfo->pFileGroup) != TSDB_CODE_SUCCESS) { + int32_t numOfBlocks = 0; + + while (pQueryHandle->pFileGroup != NULL) { + if (getFileCompInfo(pQueryHandle, &numOfBlocks, 1) != TSDB_CODE_SUCCESS) { break; } - + + assert(pCheckInfo->numOfBlocks >= 0); + // no data block in current file, try next - if (pCheckInfo->compIndex[tid].numOfSuperBlocks == 0) { - dTrace("QInfo:%p no data block in file, fid:%d, tid:%d, try next", pQueryHandle->qinfo, - pCheckInfo->pFileGroup->fileId, tid); - - pCheckInfo->pFileGroup = tsdbGetFileGroupNext(&pCheckInfo->fileIter); - continue; - } - - index = binarySearchForBlockImpl(pCheckInfo->pCompInfo->blocks, pCheckInfo->compIndex[tid].numOfSuperBlocks, pQueryHandle->order, key); - - if (type == QUERY_RANGE_GREATER_EQUAL) { - if (key <= pCheckInfo->pCompInfo->blocks[index].keyLast) { - break; - } else { - index = -1; - } - } else { - if (key >= pCheckInfo->pCompInfo->blocks[index].keyFirst) { - break; - } else { - index = -1; - } + if (pCheckInfo->numOfBlocks > 0) { + cur->fid = pQueryHandle->pFileGroup->fileId; + break; } + + dTrace("%p no data block in file, fid:%d, tid:%d, try next, %p", pQueryHandle, pQueryHandle->pFileGroup->fileId, + tid, pQueryHandle->qinfo); + + pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); } - - // failed to find qualified point in file, abort - if (index == -1) { + + if (pCheckInfo->numOfBlocks == 0) { return false; } - - assert(index >= 0 && index < pCheckInfo->compIndex[tid].numOfSuperBlocks); - - // load first data block into memory failed, caused by disk block error - bool blockLoaded = false; - SArray *sa = getDefaultLoadColumns(pQueryHandle, true); - - // todo no need to loaded at all - cur->slot = index; - + + cur->slot = 0; // always start from the first slot SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; - SCompData* data = calloc(1, sizeof(SCompData)+ sizeof(SCompCol)*pBlock->numOfCols); - - data->numOfCols = pBlock->numOfCols; - data->uid = pCheckInfo->pTableObj->tableId.uid; - - pCheckInfo->pDataCols = tdNewDataCols(1000, 2, 4096); - tdInitDataCols(pCheckInfo->pDataCols, pCheckInfo->pTableObj->schema); - - SFile* pFile = &pCheckInfo->pFileGroup->files[TSDB_FILE_TYPE_DATA]; - if (pFile->fd == FD_INITIALIZER) { - pFile->fd = open(pFile->fname, O_RDONLY); - } - - if (tsdbLoadDataBlock(pFile, pBlock, 1, pCheckInfo->pDataCols, data) == 0) { - SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; - pBlockLoadInfo->fileGroup = pCheckInfo->pFileGroup; - pBlockLoadInfo->slot = pQueryHandle->cur.slot; - pBlockLoadInfo->sid = pCheckInfo->pTableObj->tableId.tid; - - blockLoaded = true; - } - - // dError("QInfo:%p fileId:%d total numOfBlks:%d blockId:%d load into memory failed due to error in disk files", - // GET_QINFO_ADDR(pQuery), pQuery->fileId, pQuery->numOfBlocks, blkIdx); - - // failed to load data from disk, abort current query - if (blockLoaded == false) { - taosArrayDestroy(sa); - tfree(data); - - return false; - } - - // todo search qualified points in blk, according to primary key (timestamp) column - SDataCols* pDataCols = pCheckInfo->pDataCols; - - TSKEY* d = (TSKEY*) pDataCols->cols[PRIMARYKEY_TIMESTAMP_COL_INDEX].pData; - assert(d[0] == pBlock->keyFirst && d[pBlock->numOfPoints - 1] == pBlock->keyLast); - - cur->pos = binarySearchForKey(pDataCols->cols[0].pData, pBlock->numOfPoints, key, pQueryHandle->order); - - cur->fid = pCheckInfo->pFileGroup->fileId; - assert(cur->pos >= 0 && cur->fid >= 0 && cur->slot >= 0); - - filterDataInDataBlock(pQueryHandle, pCheckInfo->pDataCols, sa); - - taosArrayDestroy(sa); - tfree(data); - return pQueryHandle->realNumOfRows > 0; + return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); } -static bool hasMoreDataInFileForSingleTableModel(STsdbQueryHandle* pHandle) { +static UNUSED_FUNC bool hasMoreDataForSingleTable(STsdbQueryHandle* pHandle) { assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); - - STsdbFileH* pFileHandle = tsdbGetFile(pHandle->pTsdb); + + STsdbFileH* pFileHandle = tsdbGetFile(pHandle->pTsdb); STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - + if (!pCheckInfo->checkFirstFileBlock) { pCheckInfo->checkFirstFileBlock = true; - + if (pFileHandle != NULL) { bool found = getQualifiedDataBlock(pHandle, pCheckInfo, 1); if (found) { return true; } } - + // no data in file, try cache pHandle->cur.fid = -1; - return hasMoreDataInCacheForSingleModel(pHandle); - } else { // move to next data block in file or in cache + return hasMoreDataInCache(pHandle); + } else { // move to next data block in file or in cache return moveToNextBlock(pHandle, 1); } } -static bool hasMoreDataInCacheForMultiModel(STsdbQueryHandle* pHandle) { - size_t numOfTables = taosArrayGetSize(pHandle->pTableCheckInfo); - assert(numOfTables > 0); - - while(pHandle->activeIndex < numOfTables) { - STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - - STable *pTable = pTableCheckInfo->pTableObj; - if (pTable->mem == NULL && pTable->imem == NULL) { - pHandle->activeIndex += 1; // try next table if exits - continue; - } - - // all data in mem are checked already. - if (pTableCheckInfo->lastKey > pTable->mem->keyLast) { - pHandle->activeIndex += 1; // try next table if exits - continue; - } - - return true; +typedef struct SBlockOrderSupporter { + int32_t numOfTables; + STableBlockInfo** pDataBlockInfo; + int32_t* blockIndexArray; + int32_t* numOfBlocksPerMeter; +} SBlockOrderSupporter; + +void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t numOfTables) { + tfree(pSupporter->numOfBlocksPerMeter); + tfree(pSupporter->blockIndexArray); + + for (int32_t i = 0; i < numOfTables; ++i) { + tfree(pSupporter->pDataBlockInfo[i]); } + + tfree(pSupporter->pDataBlockInfo); +} + +static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) { + int32_t leftTableIndex = *(int32_t*)pLeft; + int32_t rightTableIndex = *(int32_t*)pRight; + + SBlockOrderSupporter* pSupporter = (SBlockOrderSupporter*)param; + + int32_t leftTableBlockIndex = pSupporter->blockIndexArray[leftTableIndex]; + int32_t rightTableBlockIndex = pSupporter->blockIndexArray[rightTableIndex]; + + if (leftTableBlockIndex > pSupporter->numOfBlocksPerMeter[leftTableIndex]) { + /* left block is empty */ + return 1; + } else if (rightTableBlockIndex > pSupporter->numOfBlocksPerMeter[rightTableIndex]) { + /* right block is empty */ + return -1; + } + + STableBlockInfo* pLeftBlockInfoEx = &pSupporter->pDataBlockInfo[leftTableIndex][leftTableBlockIndex]; + STableBlockInfo* pRightBlockInfoEx = &pSupporter->pDataBlockInfo[rightTableIndex][rightTableBlockIndex]; + + // assert(pLeftBlockInfoEx->pBlock.compBlock->offset != pRightBlockInfoEx->pBlock.compBlock->offset); + if (pLeftBlockInfoEx->pBlock.compBlock->offset == pRightBlockInfoEx->pBlock.compBlock->offset && + pLeftBlockInfoEx->pBlock.compBlock->last == pRightBlockInfoEx->pBlock.compBlock->last) { + // todo add more information + dError("error in header file, two block with same offset:%p", pLeftBlockInfoEx->pBlock.compBlock->offset); + } + + return pLeftBlockInfoEx->pBlock.compBlock->offset > pRightBlockInfoEx->pBlock.compBlock->offset ? 1 : -1; +} + +int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { + char* tmp = realloc(pQueryHandle->pDataBlockInfo, sizeof(STableBlockInfo) * numOfBlocks); + if (tmp == NULL) { + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } + + pQueryHandle->pDataBlockInfo = (STableBlockInfo*)tmp; + memset(pQueryHandle->pDataBlockInfo, 0, sizeof(STableBlockInfo) * numOfBlocks); + *numOfAllocBlocks = numOfBlocks; + + int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + + SBlockOrderSupporter sup = {0}; + sup.numOfTables = numOfTables; + sup.numOfBlocksPerMeter = calloc(1, sizeof(int32_t) * numOfTables); + sup.blockIndexArray = calloc(1, sizeof(int32_t) * numOfTables); + sup.pDataBlockInfo = calloc(1, POINTER_BYTES * numOfTables); + + if (sup.numOfBlocksPerMeter == NULL || sup.blockIndexArray == NULL || sup.pDataBlockInfo == NULL) { + cleanBlockOrderSupporter(&sup, 0); + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } + + int32_t cnt = 0; + int32_t numOfQualMeters = 0; + for (int32_t j = 0; j < numOfTables; ++j) { + STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pQueryHandle->pTableCheckInfo, j); + + SCompBlock* pBlock = pTableCheck->pCompInfo->blocks; + sup.numOfBlocksPerMeter[numOfQualMeters] = pTableCheck->numOfBlocks; + + char* buf = calloc(1, sizeof(STableBlockInfo) * pTableCheck->numOfBlocks); + if (buf == NULL) { + cleanBlockOrderSupporter(&sup, numOfQualMeters); + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } + + sup.pDataBlockInfo[numOfQualMeters] = (STableBlockInfo*)buf; + + for (int32_t k = 0; k < pTableCheck->numOfBlocks; ++k) { + STableBlockInfo* pBlockInfoEx = &sup.pDataBlockInfo[numOfQualMeters][k]; + + pBlockInfoEx->pBlock.compBlock = &pBlock[k]; + pBlockInfoEx->pBlock.fields = NULL; + + pBlockInfoEx->pTableCheckInfo = pTableCheck; + // pBlockInfoEx->groupIdx = pTableCheckInfo[j]->groupIdx; // set the group index + // pBlockInfoEx->blockIndex = pTableCheckInfo[j]->start + k; // set the block index in original meter + cnt++; + } + + numOfQualMeters++; + } + + dTrace("%p create data blocks info struct completed", pQueryHandle); + + assert(cnt <= numOfBlocks && numOfQualMeters <= numOfTables); // the pMeterDataInfo[j]->numOfBlocks may be 0 + sup.numOfTables = numOfQualMeters; + SLoserTreeInfo* pTree = NULL; + + uint8_t ret = tLoserTreeCreate(&pTree, sup.numOfTables, &sup, dataBlockOrderCompar); + if (ret != TSDB_CODE_SUCCESS) { + cleanBlockOrderSupporter(&sup, numOfTables); + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } + + int32_t numOfTotal = 0; + + while (numOfTotal < cnt) { + int32_t pos = pTree->pNode[0].index; + int32_t index = sup.blockIndexArray[pos]++; + + STableBlockInfo* pBlocksInfoEx = sup.pDataBlockInfo[pos]; + pQueryHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfoEx[index]; + + // set data block index overflow, in order to disable the offset comparator + if (sup.blockIndexArray[pos] >= sup.numOfBlocksPerMeter[pos]) { + sup.blockIndexArray[pos] = sup.numOfBlocksPerMeter[pos] + 1; + } + + tLoserTreeAdjust(pTree, pos + sup.numOfTables); + } + + /* + * available when no import exists + * for(int32_t i = 0; i < cnt - 1; ++i) { + * assert((*pDataBlockInfo)[i].pBlock.compBlock->offset < (*pDataBlockInfo)[i+1].pBlock.compBlock->offset); + * } + */ + + dTrace("%p %d data blocks sort completed", pQueryHandle, cnt); + cleanBlockOrderSupporter(&sup, numOfTables); + free(pTree); + + return TSDB_CODE_SUCCESS; +} + +static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { + STsdbFileH* pFileHandle = tsdbGetFile(pQueryHandle->pTsdb); + SQueryFilePos* cur = &pQueryHandle->cur; + + // find the start data block in file + if (!pQueryHandle->locateStart) { + pQueryHandle->locateStart = true; + + int32_t fid = getFileIdFromKey(pQueryHandle->window.skey); + + tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, TSDB_FGROUP_ITER_FORWARD); + tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); + + int32_t numOfBlocks = -1; + + // todo opt for only one table case + pQueryHandle->numOfBlocks = 0; + int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + + while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { + if (getFileCompInfo(pQueryHandle, &numOfBlocks, 1) != TSDB_CODE_SUCCESS) { + break; + } + + assert(numOfBlocks >= 0); + dTrace("%p %d blocks found in file for %d table(s), fid:%d", pQueryHandle, numOfBlocks, + pQueryHandle->pFileGroup->fileId, numOfTables); + + // todo return error code to query engine + if (createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks) != TSDB_CODE_SUCCESS) { + break; + } - // all tables has checked already - return false; + assert(numOfBlocks >= pQueryHandle->numOfBlocks); + if (pQueryHandle->numOfBlocks > 0) { + break; + } + } + + // no data in file anymore + if (pQueryHandle->numOfBlocks <= 0) { + assert(pQueryHandle->pFileGroup == NULL); + cur->fid = -1; + + return false; + } + + cur->slot = 0; + cur->fid = pQueryHandle->pFileGroup->fileId; + + STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; + STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; + SCompBlock* pBlock = pBlockInfo->pBlock.compBlock; + + return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); + } else { + if (cur->slot == pQueryHandle->numOfBlocks - 1) { // all blocks + int32_t numOfBlocks = -1; + int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + pQueryHandle->numOfBlocks = 0; + + while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { + if (getFileCompInfo(pQueryHandle, &numOfBlocks, 1) != TSDB_CODE_SUCCESS) { + break; + } + + assert(numOfBlocks >= 0); + dTrace("%p %d blocks found in file for %d table(s), fid:%d", pQueryHandle, numOfBlocks, numOfTables, + pQueryHandle->pFileGroup->fileId); + + // todo return error code to query engine + if (createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks) != TSDB_CODE_SUCCESS) { + break; + } + + assert(numOfBlocks >= pQueryHandle->numOfBlocks); + if (pQueryHandle->numOfBlocks > 0) { + break; + } + } + + // no data in file anymore + if (pQueryHandle->numOfBlocks <= 0) { + assert(pQueryHandle->pFileGroup == NULL); + cur->fid = -1; + return false; + } + + cur->slot = 0; + cur->fid = pQueryHandle->pFileGroup->fileId; + + STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; + + STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; + SCompBlock* pBlock = pBlockInfo->pBlock.compBlock; + + return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); + } else { // next block of the same file + cur->slot += 1; + cur->pos = 0; + + STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; + return loadFileDataBlock(pQueryHandle, pBlockInfo->pBlock.compBlock, pBlockInfo->pTableCheckInfo); + } + } } // handle data in cache situation -bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { - STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; - if (pHandle->model == SINGLE_TABLE_MODEL) { - return hasMoreDataInFileForSingleTableModel(pHandle); +bool tsdbNextDataBlock(tsdb_query_handle_t* pQueryHandle) { + STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; + + size_t numOfTables = taosArrayGetSize(pHandle->pTableCheckInfo); + assert(numOfTables > 0); + + if (pHandle->checkFiles) { + if (getDataBlocksInFiles(pHandle)) { + return true; + } + + pHandle->activeIndex = 0; + pHandle->checkFiles = 0; + + while (pHandle->activeIndex < numOfTables) { + if (hasMoreDataInCache(pHandle)) { + return true; + } + + pHandle->activeIndex += 1; + } + + return false; } else { - return hasMoreDataInCacheForMultiModel(pHandle); + while (pHandle->activeIndex < numOfTables) { + if (hasMoreDataInCache(pHandle)) { + return true; + } + + pHandle->activeIndex += 1; + } + + return false; } } -static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, - TSKEY* skey, TSKEY* ekey, STsdbQueryHandle* pHandle) { - int numOfRows = 0; +static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey, + STsdbQueryHandle* pHandle) { + int numOfRows = 0; int32_t numOfCols = taosArrayGetSize(pHandle->pColumns); *skey = INT64_MIN; - - while(tSkipListIterNext(pIter)) { - SSkipListNode *node = tSkipListIterGet(pIter); + + while (tSkipListIterNext(pIter)) { + SSkipListNode* node = tSkipListIterGet(pIter); if (node == NULL) break; - + SDataRow row = SL_GET_NODE_DATA(node); if (dataRowKey(row) > maxKey) break; - + if (*skey == INT64_MIN) { *skey = dataRowKey(row); } - + *ekey = dataRowKey(row); - + int32_t offset = 0; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoEx* pColInfo = taosArrayGet(pHandle->pColumns, i); - memcpy(pColInfo->pData + numOfRows*pColInfo->info.bytes, dataRowTuple(row) + offset, pColInfo->info.bytes); + memcpy(pColInfo->pData + numOfRows * pColInfo->info.bytes, dataRowTuple(row) + offset, pColInfo->info.bytes); offset += pColInfo->info.bytes; } - + numOfRows++; if (numOfRows >= maxRowsToRead) break; }; - + return numOfRows; } // copy data from cache into data block -SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { - STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; - - STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - STable *pTable = pCheckInfo->pTableObj; - - TSKEY skey = 0, ekey = 0; +SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t* pQueryHandle) { + STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; + + STable* pTable = NULL; + + TSKEY skey = 0, ekey = 0; int32_t rows = 0; - + // data in file - if (pHandle->cur.fid > 0) { - SDataBlockInfo binfo = getTrueDataBlockInfo(pHandle, pCheckInfo); - if (binfo.size == pHandle->realNumOfRows) { + if (pHandle->cur.fid >= 0) { + STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot]; + + pTable = pBlockInfo->pTableCheckInfo->pTableObj; + + SDataBlockInfo binfo = getTrueDataBlockInfo(pBlockInfo->pTableCheckInfo, pBlockInfo->pBlock.compBlock); + if (binfo.rows == pHandle->realNumOfRows) { + pBlockInfo->pTableCheckInfo->lastKey = pBlockInfo->pBlock.compBlock->keyLast + 1; return binfo; } else { /* not a whole disk block, only the qualified rows, so this block is loaded in to buffer during the * block next function */ SColumnInfoEx* pColInfoEx = taosArrayGet(pHandle->pColumns, 0); - + rows = pHandle->realNumOfRows; - skey = *(TSKEY*) pColInfoEx->pData; - ekey = *(TSKEY*) ((char*)pColInfoEx->pData + TSDB_KEYSIZE * (rows - 1)); + skey = *(TSKEY*)pColInfoEx->pData; + ekey = *(TSKEY*)((char*)pColInfoEx->pData + TSDB_KEYSIZE * (rows - 1)); + + // update the last key value + pBlockInfo->pTableCheckInfo->lastKey = ekey + 1; } } else { + STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + pTable = pCheckInfo->pTableObj; + if (pTable->mem != NULL) { // create mem table iterator if it is not created yet if (pCheckInfo->iter == NULL) { pCheckInfo->iter = tSkipListCreateIter(pTable->mem->pData); } rows = tsdbReadRowsFromCache(pCheckInfo->iter, INT64_MAX, 2, &skey, &ekey, pHandle); + + // update the last key value + pCheckInfo->lastKey = ekey + 1; } } - + SDataBlockInfo blockInfo = { - .uid = pTable->tableId.uid, - .sid = pTable->tableId.tid, - .size = rows, - .window = {.skey = skey, .ekey = ekey} - }; - - // update the last key value - pCheckInfo->lastKey = ekey + 1; + .uid = pTable->tableId.uid, .sid = pTable->tableId.tid, .rows = rows, .window = {.skey = skey, .ekey = ekey}}; + return blockInfo; } // return null for data block in cache -int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t *pQueryHandle, SDataStatis **pBlockStatis) { +int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t* pQueryHandle, SDataStatis** pBlockStatis) { *pBlockStatis = NULL; return TSDB_CODE_SUCCESS; } -SArray *tsdbRetrieveDataBlock(tsdb_query_handle_t *pQueryHandle, SArray *pIdList) { +SArray* tsdbRetrieveDataBlock(tsdb_query_handle_t* pQueryHandle, SArray* pIdList) { /** * In the following two cases, the data has been loaded to SColumnInfoEx. * 1. data is from cache, 2. data block is not completed qualified to query time range */ - STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; - + STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; + if (pHandle->cur.fid < 0) { return pHandle->pColumns; } else { - STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - - SDataBlockInfo binfo = getTrueDataBlockInfo(pHandle, pCheckInfo); - assert(pHandle->realNumOfRows <= binfo.size); - - if (pHandle->realNumOfRows < binfo.size) { + STableBlockInfo* pBlockInfoEx = &pHandle->pDataBlockInfo[pHandle->cur.slot]; + STableCheckInfo* pCheckInfo = pBlockInfoEx->pTableCheckInfo; + + SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlockInfoEx->pBlock.compBlock); + assert(pHandle->realNumOfRows <= binfo.rows); + + if (pHandle->realNumOfRows < binfo.rows) { return pHandle->pColumns; } else { - SArray *sa = getDefaultLoadColumns(pHandle, true); - // data block has been loaded, todo extract method SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo; if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->sid == pCheckInfo->pTableObj->tableId.tid) { return pHandle->pColumns; } else { - doLoadDataFromFileBlock(pHandle); - filterDataInDataBlock(pHandle, pCheckInfo->pDataCols, sa); - + SCompBlock* pBlock = pBlockInfoEx->pBlock.compBlock; + doLoadFileDataBlock(pHandle, pBlock, pCheckInfo); + + SArray* sa = getDefaultLoadColumns(pHandle, true); + filterDataInDataBlock(pHandle, pCheckInfo, pBlock, sa); + taosArrayDestroy(sa); + return pHandle->pColumns; } } } } -int32_t tsdbResetQuery(tsdb_query_handle_t *pQueryHandle, STimeWindow *window, tsdbpos_t position, int16_t order) { +int32_t tsdbResetQuery(tsdb_query_handle_t* pQueryHandle, STimeWindow* window, tsdbpos_t position, int16_t order) { return 0; } -int32_t tsdbDataBlockSeek(tsdb_query_handle_t *pQueryHandle, tsdbpos_t pos) { return 0;} +int32_t tsdbDataBlockSeek(tsdb_query_handle_t* pQueryHandle, tsdbpos_t pos) { return 0; } -tsdbpos_t tsdbDataBlockTell(tsdb_query_handle_t *pQueryHandle) { return NULL; } +tsdbpos_t tsdbDataBlockTell(tsdb_query_handle_t* pQueryHandle) { return NULL; } -SArray *tsdbRetrieveDataRow(tsdb_query_handle_t *pQueryHandle, SArray *pIdList, SQueryRowCond *pCond) { return NULL;} +SArray* tsdbRetrieveDataRow(tsdb_query_handle_t* pQueryHandle, SArray* pIdList, SQueryRowCond* pCond) { return NULL; } -tsdb_query_handle_t *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stableId, const char *pTagFilterStr) { +tsdb_query_handle_t* tsdbQueryFromTagConds(STsdbQueryCond* pCond, int16_t stableId, const char* pTagFilterStr) { return NULL; } -SArray *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle) { return NULL; } +SArray* tsdbGetTableList(tsdb_query_handle_t* pQueryHandle) { return NULL; } -static SArray* createTableIdArrayList(STsdbRepo* tsdb, int64_t uid) { +static int32_t getAllTableIdList(STsdbRepo* tsdb, int64_t uid, SArray* list) { STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); - assert(pTable != NULL); //assert pTable is a super table - - size_t size = tSkipListGetSize(pTable->pIndex); - SArray* pList = taosArrayInit(size, sizeof(STableId)); - + assert(pTable != NULL); // assert pTable is a super table + SSkipListIterator* iter = tSkipListCreateIter(pTable->pIndex); - while(tSkipListIterNext(iter)) { + while (tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); - STable* t = *(STable**) SL_GET_NODE_DATA(pNode); - - taosArrayPush(pList, &t->tableId); + STable* t = *(STable**)SL_GET_NODE_DATA(pNode); + + taosArrayPush(list, &t->tableId); } - - return pList; + + return TSDB_CODE_SUCCESS; } -typedef struct SSyntaxTreeFilterSupporter { +typedef struct SExprTreeSupporter { SSchema* pTagSchema; int32_t numOfTags; int32_t optr; -} SSyntaxTreeFilterSupporter; +} SExprTreeSupporter; /** - * convert the result pointer to STabObj instead of tSkipListNode + * convert the result pointer to table id instead of table object pointer * @param pRes */ -static UNUSED_FUNC void tansformQueryResult(SArray* pRes) { - if (pRes == NULL || taosArrayGetSize(pRes) == 0) { +static void convertQueryResult(SArray* pRes, SArray* pTableList) { + if (pTableList == NULL || taosArrayGetSize(pTableList) == 0) { return; } - - size_t size = taosArrayGetSize(pRes); + + size_t size = taosArrayGetSize(pTableList); for (int32_t i = 0; i < size; ++i) { -// pRes->pRes[i] = ((tSkipListNode*)(pRes->pRes[i]))->pData; + STable* pTable = taosArrayGetP(pTableList, i); + taosArrayPush(pRes, &pTable->tableId); } } -void tSQLListTraverseDestroyInfo(void* param) { +void destroyHelper(void* param) { if (param == NULL) { return; } - + tQueryInfo* pInfo = (tQueryInfo*)param; tVariantDestroy(&(pInfo->q)); free(param); } -static char* convertTagQueryStr(const wchar_t* str, size_t len) { - char* mbs = NULL; - - if (len > 0) { - mbs = calloc(1, (len + 1) * TSDB_NCHAR_SIZE); - taosUcs4ToMbs((void*) str, len * TSDB_NCHAR_SIZE, mbs); //todo add log - } - - return mbs; -} - -static int32_t compareStrVal(const void* pLeft, const void* pRight) { - int32_t ret = strcmp(pLeft, pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -static int32_t compareWStrVal(const void* pLeft, const void* pRight) { - int32_t ret = wcscmp(pLeft, pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -static int32_t compareIntVal(const void* pLeft, const void* pRight) { - DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_INT64_VAL(pRight)); -} - -static int32_t compareIntDoubleVal(const void* pLeft, const void* pRight) { - DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_DOUBLE_VAL(pRight)); -} - -static int32_t compareDoubleVal(const void* pLeft, const void* pRight) { - DEFAULT_COMP(GET_DOUBLE_VAL(pLeft), GET_DOUBLE_VAL(pRight)); -} - -static int32_t compareDoubleIntVal(const void* pLeft, const void* pRight) { - double ret = (*(double*)pLeft) - (*(int64_t*)pRight); - if (fabs(ret) < DBL_EPSILON) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { +static UNUSED_FUNC int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; - + const char* pattern = pRight; const char* str = pLeft; - + int32_t ret = patternMatch(pattern, str, strlen(str), &pInfo); - + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } -static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { +static UNUSED_FUNC int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; - + const wchar_t* pattern = pRight; const wchar_t* str = pLeft; - + int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo); - + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } -static __compar_fn_t getFilterComparator(int32_t type, int32_t filterType, int32_t optr) { - __compar_fn_t comparator = NULL; - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_BOOL: { - if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { - comparator = compareIntVal; - } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { - comparator = compareIntDoubleVal; - } - break; - } - - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: { - if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { - comparator = compareDoubleIntVal; - } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { - comparator = compareDoubleVal; - } - break; - } - - case TSDB_DATA_TYPE_BINARY: { - assert(filterType == TSDB_DATA_TYPE_BINARY); - - if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ - comparator = compareStrPatternComp; - } else { /* normal relational comparator */ - comparator = compareStrVal; - } - - break; - } - - case TSDB_DATA_TYPE_NCHAR: { - assert(filterType == TSDB_DATA_TYPE_NCHAR); - - if (optr == TSDB_RELATION_LIKE) { - comparator = compareWStrPatternComp; - } else { - comparator = compareWStrVal; - } - - break; - } - default: - comparator = compareIntVal; - break; - } - - return comparator; -} +// static __compar_fn_t getFilterComparator(int32_t type, int32_t filterType, int32_t optr) { +// __compar_fn_t comparator = NULL; +// +// switch (type) { +// case TSDB_DATA_TYPE_TINYINT: +// case TSDB_DATA_TYPE_SMALLINT: +// case TSDB_DATA_TYPE_INT: +// case TSDB_DATA_TYPE_BIGINT: +// case TSDB_DATA_TYPE_BOOL: { +// if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { +// comparator = compareIntVal; +// } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { +// comparator = compareIntDoubleVal; +// } +// break; +// } +// +// case TSDB_DATA_TYPE_FLOAT: +// case TSDB_DATA_TYPE_DOUBLE: { +// if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { +// comparator = compareDoubleIntVal; +// } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { +// comparator = compareDoubleVal; +// } +// break; +// } +// +// case TSDB_DATA_TYPE_BINARY: { +// assert(filterType == TSDB_DATA_TYPE_BINARY); +// +// if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ +// comparator = compareStrPatternComp; +// } else { /* normal relational comparator */ +// comparator = compareStrVal; +// } +// +// break; +// } +// +// case TSDB_DATA_TYPE_NCHAR: { +// assert(filterType == TSDB_DATA_TYPE_NCHAR); +// +// if (optr == TSDB_RELATION_LIKE) { +// comparator = compareWStrPatternComp; +// } else { +// comparator = compareWStrVal; +// } +// +// break; +// } +// default: +// comparator = compareIntVal; +// break; +// } +// +// return comparator; +//} -static void getTagColumnInfo(SSyntaxTreeFilterSupporter* pSupporter, SSchema* pSchema, int32_t* index, - int32_t* offset) { +static void getTagColumnInfo(SExprTreeSupporter* pSupporter, SSchema* pSchema, int32_t* index, int32_t* offset) { *index = 0; *offset = 0; - + // filter on table name(TBNAME) if (strcasecmp(pSchema->name, TSQL_TBNAME_L) == 0) { *index = TSDB_TBNAME_COLUMN_INDEX; *offset = TSDB_TBNAME_COLUMN_INDEX; return; } - + while ((*index) < pSupporter->numOfTags) { if (pSupporter->pTagSchema[*index].bytes == pSchema->bytes && pSupporter->pTagSchema[*index].type == pSchema->type && - strcmp(pSupporter->pTagSchema[*index].name, pSchema->name) == 0) { + pSupporter->pTagSchema[*index].colId == pSchema->colId) { break; } else { (*offset) += pSupporter->pTagSchema[(*index)++].bytes; @@ -1174,53 +1342,53 @@ static void getTagColumnInfo(SSyntaxTreeFilterSupporter* pSupporter, SSchema* pS } void filterPrepare(void* expr, void* param) { - tExprNode *pExpr = (tExprNode*) expr; + tExprNode* pExpr = (tExprNode*)expr; if (pExpr->_node.info != NULL) { return; } - + int32_t i = 0, offset = 0; pExpr->_node.info = calloc(1, sizeof(tQueryInfo)); - + tQueryInfo* pInfo = pExpr->_node.info; - - SSyntaxTreeFilterSupporter* pSupporter = (SSyntaxTreeFilterSupporter*)param; - + + SExprTreeSupporter* pSupporter = (SExprTreeSupporter*)param; + tVariant* pCond = pExpr->_node.pRight->pVal; SSchema* pSchema = pExpr->_node.pLeft->pSchema; - + getTagColumnInfo(pSupporter, pSchema, &i, &offset); assert((i >= 0 && i < TSDB_MAX_TAGS) || (i == TSDB_TBNAME_COLUMN_INDEX)); assert((offset >= 0 && offset < TSDB_MAX_TAGS_LEN) || (offset == TSDB_TBNAME_COLUMN_INDEX)); - + pInfo->sch = *pSchema; - pInfo->colIdx = i; + pInfo->colIndex = i; pInfo->optr = pExpr->_node.optr; pInfo->offset = offset; - pInfo->compare = getFilterComparator(pSchema->type, pCond->nType, pInfo->optr); - + // pInfo->compare = getFilterComparator(pSchema->type, pCond->nType, pInfo->optr); + tVariantAssign(&pInfo->q, pCond); tVariantTypeSetType(&pInfo->q, pInfo->sch.type); } bool tSkipListNodeFilterCallback(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*)param; - + STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); - - char* val = dataRowTuple(pTable->tagVal); // todo not only the first column + + char* val = dataRowTuple(pTable->tagVal); // todo not only the first column int8_t type = pInfo->sch.type; - + int32_t ret = 0; if (pInfo->q.nType == TSDB_DATA_TYPE_BINARY || pInfo->q.nType == TSDB_DATA_TYPE_NCHAR) { ret = pInfo->compare(val, pInfo->q.pz); } else { tVariant t = {0}; - tVariantCreateFromBinary(&t, val, (uint32_t) pInfo->sch.bytes, type); - + tVariantCreateFromBinary(&t, val, (uint32_t)pInfo->sch.bytes, type); + ret = pInfo->compare(&t.i64Key, &pInfo->q.i64Key); } - + switch (pInfo->optr) { case TSDB_RELATION_EQUAL: { return ret == 0; @@ -1228,10 +1396,10 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { case TSDB_RELATION_NOT_EQUAL: { return ret != 0; } - case TSDB_RELATION_LARGE_EQUAL: { + case TSDB_RELATION_GREATER_EQUAL: { return ret >= 0; } - case TSDB_RELATION_LARGE: { + case TSDB_RELATION_GREATER: { return ret > 0; } case TSDB_RELATION_LESS_EQUAL: { @@ -1243,90 +1411,85 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { case TSDB_RELATION_LIKE: { return ret == 0; } - + default: assert(false); } return true; } -static int32_t doQueryTableList(STable* pSTable, SArray* pRes, const char* pCond) { -// STColumn* stcol = schemaColAt(pSTable->tagSchema, 0); - - tExprNode* pExpr = NULL; -// tSQLBinaryExprFromString(&pExpr, stcol, schemaNCols(pSTable->tagSchema), (char*) pCond, strlen(pCond)); - - // failed to build expression, no result, return immediately - if (pExpr == NULL) { - mError("table:%" PRIu64 ", no result returned, error in super table query expression:%s", pSTable->tableId.uid, pCond); - tfree(pCond); - - return TSDB_CODE_OPS_NOT_SUPPORT; - } - +static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) { // query according to the binary expression -// SSyntaxTreeFilterSupporter s = {.pTagSchema = stcol, .numOfTags = schemaNCols(pSTable->tagSchema)}; -// -// SBinaryFilterSupp supp = { -// .fp = (__result_filter_fn_t)tSkipListNodeFilterCallback, -// .setupInfoFn = (__do_filter_suppl_fn_t)filterPrepare, -// .pExtInfo = &s -// }; -// -// tSQLBinaryExprTraverse(pExpr, pSTable->pIndex, pRes, &supp); -// tExprTreeDestroy(&pExpr, tSQLListTraverseDestroyInfo); -// -// tansformQueryResult(pRes); - + STSchema* pSchema = pSTable->tagSchema; + SSchema* schema = calloc(schemaNCols(pSchema), sizeof(SSchema)); + for (int32_t i = 0; i < schemaNCols(pSchema); ++i) { + schema[i].colId = schemaColAt(pSchema, i)->colId; + schema[i].type = schemaColAt(pSchema, i)->type; + schema[i].bytes = schemaColAt(pSchema, i)->bytes; + } + + SExprTreeSupporter s = {.pTagSchema = schema, .numOfTags = schemaNCols(pSTable->tagSchema)}; + + SBinaryFilterSupp supp = { + .fp = (__result_filter_fn_t)tSkipListNodeFilterCallback, .setupInfoFn = filterPrepare, .pExtInfo = &s}; + + SArray* pTableList = taosArrayInit(8, POINTER_BYTES); + + tExprTreeTraverse(pExpr, pSTable->pIndex, pTableList, &supp); + tExprTreeDestroy(&pExpr, destroyHelper); + + convertQueryResult(pRes, pTableList); return TSDB_CODE_SUCCESS; } -SArray *tsdbQueryTableList(tsdb_repo_t* tsdb, int64_t uid, const wchar_t *pTagCond, size_t len) { - // no condition, all tables created according to the stable will involved in querying - SArray* result = taosArrayInit(8, POINTER_BYTES); - - if (pTagCond == NULL || wcslen(pTagCond) == 0) { - return createTableIdArrayList(tsdb, uid); - } else { - char* str = convertTagQueryStr(pTagCond, len); - - STable* pSTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); - assert(pSTable != NULL); - - doQueryTableList(pSTable, result, str); - return result; +int32_t tsdbQueryTags(tsdb_repo_t* tsdb, int64_t uid, const char* pTagCond, size_t len, SArray* res) { + if (pTagCond == NULL || len == 0) { // no condition, all tables created according to this stable are involved + return getAllTableIdList(tsdb, uid, res); } + + STable* pSTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); + assert(pSTable != NULL); + + tExprNode* pExprNode = NULL; + int32_t ret = TSDB_CODE_SUCCESS; + + // failed to build expression, no result, return immediately + if ((ret = exprTreeFromBinary(pTagCond, len, &pExprNode) != TSDB_CODE_SUCCESS) || (pExprNode == NULL)) { + dError("stable:%" PRIu64 ", failed to deserialize expression tree, error exists", uid); + return ret; + } + + return doQueryTableList(pSTable, res, pExprNode); } void tsdbCleanupQueryHandle(tsdb_query_handle_t queryHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) queryHandle; - + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle; + size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - for(int32_t i = 0; i < size; ++i) { + for (int32_t i = 0; i < size; ++i) { STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); tSkipListDestroyIter(pTableCheckInfo->iter); if (pTableCheckInfo->pDataCols != NULL) { tfree(pTableCheckInfo->pDataCols->buf); } - + tfree(pTableCheckInfo->pDataCols); - + tfree(pTableCheckInfo->pCompInfo); - tfree(pTableCheckInfo->compIndex); } - + taosArrayDestroy(pQueryHandle->pTableCheckInfo); - + tfree(pQueryHandle->compIndex); + size_t cols = taosArrayGetSize(pQueryHandle->pColumns); - for(int32_t i = 0; i < cols; ++i) { - SColumnInfoEx *pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + for (int32_t i = 0; i < cols; ++i) { + SColumnInfoEx* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); tfree(pColInfo->pData); } - + taosArrayDestroy(pQueryHandle->pColumns); - tfree(pQueryHandle->unzipBuffer); - tfree(pQueryHandle->secondaryUnzipBuffer); + tfree(pQueryHandle->pDataBlockInfo); tfree(pQueryHandle); } From 4241da56e1b26091573656eb61d511066bfcc9a4 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 10 Apr 2020 23:16:56 +0800 Subject: [PATCH 39/78] [td-98] update macro --- src/query/tests/astTest.cpp | 12 +++++++----- src/query/tests/tsBufTest.cpp | 18 +++++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/query/tests/astTest.cpp b/src/query/tests/astTest.cpp index 3b1f36e90f..0f41ebea26 100644 --- a/src/query/tests/astTest.cpp +++ b/src/query/tests/astTest.cpp @@ -255,7 +255,7 @@ static void testQueryStr(SSchema *schema, int32_t numOfCols, char *sql, tSkipLis printf("expr is: %s\n", str); SArray *result = NULL; - // tSQLBinaryExprTraverse(pExpr, pSkipList, result, tSkipListNodeFilterCallback, &result); + // tExprTreeTraverse(pExpr, pSkipList, result, tSkipListNodeFilterCallback, &result); // printf("the result is:%lld\n", result.num); // // bool findResult = false; @@ -533,7 +533,7 @@ tExprNode* createExpr2() { auto *p2 = (tExprNode*) calloc(1, sizeof(tExprNode)); p2->nodeType = TSQL_NODE_EXPR; - p2->_node.optr = TSDB_RELATION_LARGE_EQUAL; + p2->_node.optr = TSDB_RELATION_GREATER_EQUAL; p2->_node.pLeft = pLeft1; p2->_node.pRight = pRight1; p2->_node.hasPK = false; @@ -556,7 +556,8 @@ void exprSerializeTest1() { ASSERT_TRUE(size > 0); char* b = tbufGetData(&buf, false); - tExprNode* p2 = exprTreeFromBinary(b, size); + tExprNode* p2 = NULL; + exprTreeFromBinary(b, size, &p2); ASSERT_EQ(p1->nodeType, p2->nodeType); ASSERT_EQ(p2->_node.optr, p1->_node.optr); @@ -592,7 +593,8 @@ void exprSerializeTest2() { ASSERT_TRUE(size > 0); char* b = tbufGetData(&buf, false); - tExprNode* p2 = exprTreeFromBinary(b, size); + tExprNode* p2 = NULL; + exprTreeFromBinary(b, size, &p2); ASSERT_EQ(p1->nodeType, p2->nodeType); ASSERT_EQ(p2->_node.optr, p1->_node.optr); @@ -617,7 +619,7 @@ void exprSerializeTest2() { ASSERT_EQ(c1Right->nodeType, c2Right->nodeType); ASSERT_EQ(c2Right->nodeType, TSQL_NODE_EXPR); - ASSERT_EQ(c2Right->_node.optr, TSDB_RELATION_LARGE_EQUAL); + ASSERT_EQ(c2Right->_node.optr, TSDB_RELATION_GREATER_EQUAL); ASSERT_EQ(c2Right->_node.pRight->pVal->dKey, 91.99); ASSERT_EQ(p2->_node.hasPK, true); diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index ff58e2a974..f50e1a5a71 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -37,7 +37,7 @@ void simpleTest() { int64_t* list = createTsList(10, 10000000, 30); tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num); EXPECT_EQ(pTSBuf->block.tag, tag); @@ -65,7 +65,7 @@ void largeTSTest() { EXPECT_EQ(pTSBuf->tsData.len, 0); EXPECT_EQ(pTSBuf->block.tag, tag); EXPECT_EQ(pTSBuf->numOfVnodes, 1); - EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -91,7 +91,7 @@ void multiTagsTest() { start += step * num; } - EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); @@ -127,7 +127,7 @@ void multiVnodeTagsTest() { EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); } - EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); @@ -167,7 +167,7 @@ void loadDataTest() { EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); } - EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); @@ -252,7 +252,7 @@ void TSTraverse() { int64_t s = taosGetTimestampUs(); printf("start:%" PRIu64 "\n", s); - pTSBuf->cur.order = TSQL_SO_DESC; + pTSBuf->cur.order = TSDB_ORDER_DESC; // complete reverse traverse int32_t x = 0; @@ -263,7 +263,7 @@ void TSTraverse() { // specify the data block with vnode and tags value tsBufResetPos(pTSBuf); - pTSBuf->cur.order = TSQL_SO_DESC; + pTSBuf->cur.order = TSDB_ORDER_DESC; int32_t startVnode = 1; int32_t startTag = 2; @@ -297,7 +297,7 @@ void TSTraverse() { ///////////////////////////////////////////////////////////////////////////////// // traverse - pTSBuf->cur.order = TSQL_SO_ASC; + pTSBuf->cur.order = TSDB_ORDER_ASC; tsBufResetPos(pTSBuf); // complete forwards traverse @@ -308,7 +308,7 @@ void TSTraverse() { // specify the data block with vnode and tags value tsBufResetPos(pTSBuf); - pTSBuf->cur.order = TSQL_SO_ASC; + pTSBuf->cur.order = TSDB_ORDER_ASC; startVnode = 1; startTag = 2; From 78f7134ef1547176034b97b248aa37f01ccfebf1 Mon Sep 17 00:00:00 2001 From: slguan Date: Sat, 11 Apr 2020 00:09:37 +0800 Subject: [PATCH 40/78] [TD-52] rename some defs --- cmake/define.inc | 4 ++-- src/dnode/src/dnodeMgmt.c | 2 +- src/inc/mnode.h | 27 ++++++++-------------- src/{mnode/inc/mgmtMnode.h => inc/mpeer.h} | 18 +++++++-------- src/inc/taosdef.h | 4 ++-- src/inc/taosmsg.h | 2 +- src/mnode/src/mgmtDb.c | 4 ++-- src/mnode/src/mgmtDnode.c | 6 ++--- src/mnode/src/mgmtMain.c | 12 +++++----- src/mnode/src/mgmtMnode.c | 20 ++++++++-------- src/mnode/src/mgmtProfile.c | 8 +++---- src/mnode/src/mgmtShell.c | 14 +++++------ src/mnode/src/mgmtTable.c | 4 +--- src/mnode/src/mgmtUser.c | 8 +------ src/mnode/src/mgmtVgroup.c | 4 +--- 15 files changed, 58 insertions(+), 79 deletions(-) rename src/{mnode/inc/mgmtMnode.h => inc/mpeer.h} (74%) diff --git a/cmake/define.inc b/cmake/define.inc index e43e570979..5f17ee1216 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -11,9 +11,9 @@ ENDIF () IF (TD_VPEER) ADD_DEFINITIONS(-D_VPEER) - ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=3) + #ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=3) ELSE () - ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=1) + #ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=1) ENDIF () IF (TD_ACCOUNT) diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index da80206e4c..0ec769c0af 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -45,7 +45,7 @@ static void *tsDnodeTmr = NULL; static void *tsStatusTimer = NULL; static uint32_t tsRebootTime; static int32_t tsDnodeId = 0; -static char tsDnodeName[TSDB_DNODE_NAME_LEN]; +static char tsDnodeName[TSDB_NODE_NAME_LEN]; int32_t dnodeInitMgmt() { dnodeReadDnodeId(); diff --git a/src/inc/mnode.h b/src/inc/mnode.h index dec9292209..b19e91a90e 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -41,28 +41,19 @@ struct _vg_obj; struct _db_obj; struct _acct_obj; struct _user_obj; +struct _mnode_obj; -typedef struct { +typedef struct _mnode_obj { int32_t mnodeId; - uint32_t privateIp; - uint32_t publicIp; + int32_t dnodeId; int64_t createdTime; - int64_t lostTime; - uint64_t dbVersion; - uint32_t rack; - uint16_t idc; - uint16_t slot; - int8_t role; - int8_t status; - int8_t numOfMnodes; - int32_t numOfDnodes; - char mnodeName[TSDB_DNODE_NAME_LEN + 1]; - int8_t reserved[15]; + int8_t reserved[14]; int8_t updateEnd[1]; int32_t refCount; - int syncFd; - void *hbTimer; - void *pSync; + int8_t role; + int8_t status; + uint32_t privateIp; + uint32_t publicIp; } SMnodeObj; typedef struct _dnode_obj { @@ -81,7 +72,7 @@ typedef struct _dnode_obj { int8_t alternativeRole; // from dnode status msg, 0-any, 1-mgmt, 2-dnode int8_t status; // set in balance function int32_t customScore; // config by user - char dnodeName[TSDB_DNODE_NAME_LEN + 1]; + char dnodeName[TSDB_NODE_NAME_LEN + 1]; int8_t reserved[15]; int8_t updateEnd[1]; int32_t refCount; diff --git a/src/mnode/inc/mgmtMnode.h b/src/inc/mpeer.h similarity index 74% rename from src/mnode/inc/mgmtMnode.h rename to src/inc/mpeer.h index ad9688c0ee..7007f19226 100644 --- a/src/mnode/inc/mgmtMnode.h +++ b/src/inc/mpeer.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_MGMT_MNODE_H -#define TDENGINE_MGMT_MNODE_H +#ifndef TDENGINE_MPEER_H +#define TDENGINE_MPEER_H #ifdef __cplusplus extern "C" { @@ -33,15 +33,15 @@ enum _TSDB_MN_ROLE { TSDB_MN_ROLE_MASTER }; -int32_t mgmtInitMnodes(); -void mgmtCleanupMnodes(); +int32_t mpeerInit(); +void mpeerCleanup(); -bool mgmtInServerStatus(); -bool mgmtIsMaster(); +bool mpeerInServerStatus(); +bool mpeerIsMaster(); -bool mgmtCheckRedirect(void *handle); -void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet); -void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet); +bool mpeerCheckRedirect(void *handle); +void mpeerGetPrivateIpList(SRpcIpSet *ipSet); +void mpeerGetPublicIpList(SRpcIpSet *ipSet); #ifdef __cplusplus } diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index d6a9447e3d..85b31a2659 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -176,7 +176,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_COLUMNS 256 #define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns -#define TSDB_DNODE_NAME_LEN 64 +#define TSDB_NODE_NAME_LEN 64 #define TSDB_TABLE_NAME_LEN 192 #define TSDB_DB_NAME_LEN 32 #define TSDB_COL_NAME_LEN 64 @@ -233,7 +233,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); /* * this is defined in CMakeList.txt */ -//#define TSDB_REPLICA_MAX_NUM 3 +#define TSDB_REPLICA_MAX_NUM 3 #define TSDB_TBNAME_COLUMN_INDEX (-1) #define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index f5168a2c9e..09d306ffad 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -579,7 +579,7 @@ typedef struct { typedef struct { uint32_t version; int32_t dnodeId; - char dnodeName[TSDB_DNODE_NAME_LEN]; + char dnodeName[TSDB_NODE_NAME_LEN + 1]; uint32_t privateIp; uint32_t publicIp; uint32_t lastReboot; // time stamp for last reboot diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 9ecc2c6458..0f18de3253 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -24,8 +24,8 @@ #include "mgmtDb.h" #include "tcluster.h" #include "tgrant.h" +#include "mpeer.h" #include "mgmtShell.h" -#include "mgmtMnode.h" #include "mgmtProfile.h" #include "mgmtSdb.h" #include "mgmtTable.h" @@ -678,7 +678,7 @@ static int32_t mgmtSetDbDropping(SDbObj *pDb) { } static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { - if (mgmtCheckRedirect(pMsg->thandle)) return; + if (mpeerCheckRedirect(pMsg->thandle)) return; SCMCreateDbMsg *pCreate = pMsg->pCont; pCreate->maxSessions = htonl(pCreate->maxSessions); diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 2f4abc5cfe..1d32ded088 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -19,8 +19,8 @@ #include "tbalance.h" #include "tcluster.h" #include "mnode.h" +#include "mpeer.h" #include "mgmtDClient.h" -#include "mgmtMnode.h" #include "mgmtShell.h" #include "mgmtDServer.h" #include "mgmtUser.h" @@ -141,7 +141,7 @@ static void clusterProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) { } void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { - if (mgmtCheckRedirect(rpcMsg->handle)) return; + if (mpeerCheckRedirect(rpcMsg->handle)) return; SDMStatusMsg *pStatus = rpcMsg->pCont; pStatus->dnodeId = htonl(pStatus->dnodeId); @@ -221,7 +221,7 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { return; } - mgmtGetMnodePrivateIpList(&pRsp->ipList); + mpeerGetPrivateIpList(&pRsp->ipList); pRsp->dnodeState.dnodeId = htonl(pDnode->dnodeId); pRsp->dnodeState.moduleStatus = htonl(pDnode->moduleStatus); diff --git a/src/mnode/src/mgmtMain.c b/src/mnode/src/mgmtMain.c index e04630d745..cb3c9fae1e 100644 --- a/src/mnode/src/mgmtMain.c +++ b/src/mnode/src/mgmtMain.c @@ -22,11 +22,11 @@ #include "taccount.h" #include "tbalance.h" #include "tcluster.h" +#include "tgrant.h" +#include "mpeer.h" #include "mgmtDb.h" #include "mgmtDClient.h" #include "mgmtDServer.h" -#include "tgrant.h" -#include "mgmtMnode.h" #include "mgmtSdb.h" #include "mgmtVgroup.h" #include "mgmtUser.h" @@ -117,8 +117,8 @@ int32_t mgmtStartSystem() { return -1; } - if (mgmtInitMnodes() < 0) { - mError("failed to init mnodes"); + if (mpeerInit() < 0) { + mError("failed to init mpeers"); return -1; } @@ -135,7 +135,7 @@ int32_t mgmtStartSystem() { void mgmtStopSystem() { - if (mgmtIsMaster()) { + if (mpeerIsMaster()) { mTrace("it is a master mgmt node, it could not be stopped"); return; } @@ -147,7 +147,7 @@ void mgmtStopSystem() { void mgmtCleanUpSystem() { mPrint("starting to clean up mgmt"); grantCleanUp(); - mgmtCleanupMnodes(); + mpeerCleanup(); balanceCleanUp(); mgmtCleanUpShell(); mgmtCleanupDClient(); diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index 1e2a4e9066..c349969019 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -17,7 +17,7 @@ #include "os.h" #include "taoserror.h" #include "trpc.h" -#include "mgmtMnode.h" +#include "mpeer.h" #include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtUser.h" @@ -28,7 +28,7 @@ static SMnodeObj tsMnodeObj = {0}; static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); -int32_t mgmtInitMnodes() { +int32_t mpeerInit() { mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mgmtGetMnodeMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mgmtRetrieveMnodes); @@ -38,16 +38,14 @@ int32_t mgmtInitMnodes() { tsMnodeObj.createdTime = taosGetTimestampMs(); tsMnodeObj.role = TSDB_MN_ROLE_MASTER; tsMnodeObj.status = TSDB_MN_STATUS_SERVING; - tsMnodeObj.numOfMnodes = 1; - sprintf(tsMnodeObj.mnodeName, "%d", tsMnodeObj.mnodeId); - + return TSDB_CODE_SUCCESS; } -void mgmtCleanupMnodes() {} -bool mgmtInServerStatus() { return tsMnodeObj.status == TSDB_MN_STATUS_SERVING; } -bool mgmtIsMaster() { return tsMnodeObj.role == TSDB_MN_ROLE_MASTER; } -bool mgmtCheckRedirect(void *thandle) { return false; } +void mpeerCleanup() {} +bool mpeerInServerStatus() { return tsMnodeObj.status == TSDB_MN_STATUS_SERVING; } +bool mpeerIsMaster() { return tsMnodeObj.role == TSDB_MN_ROLE_MASTER; } +bool mpeerCheckRedirect(void *thandle) { return false; } static int32_t mgmtGetMnodesNum() { return 1; @@ -190,14 +188,14 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi return numOfRows; } -void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet) { +void mpeerGetPrivateIpList(SRpcIpSet *ipSet) { ipSet->inUse = 0; ipSet->port = htons(tsMnodeDnodePort); ipSet->numOfIps = 1; ipSet->ip[0] = htonl(tsMnodeObj.privateIp); } -void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet) { +void mpeerGetPublicIpList(SRpcIpSet *ipSet) { ipSet->inUse = 0; ipSet->port = htons(tsMnodeDnodePort); ipSet->numOfIps = 1; diff --git a/src/mnode/src/mgmtProfile.c b/src/mnode/src/mgmtProfile.c index 2b22fae47a..36481b81b0 100644 --- a/src/mnode/src/mgmtProfile.c +++ b/src/mnode/src/mgmtProfile.c @@ -19,7 +19,7 @@ #include "taccount.h" #include "tcluster.h" #include "mgmtDb.h" -#include "mgmtMnode.h" +#include "mpeer.h" #include "mgmtProfile.h" #include "mgmtShell.h" #include "mgmtTable.h" @@ -681,7 +681,7 @@ int32_t mgmtRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; - if (mgmtCheckRedirect(pMsg->thandle)) return; + if (mpeerCheckRedirect(pMsg->thandle)) return; SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { @@ -705,7 +705,7 @@ void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) { void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; - if (mgmtCheckRedirect(pMsg->thandle)) return; + if (mpeerCheckRedirect(pMsg->thandle)) return; SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { @@ -729,7 +729,7 @@ void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) { void mgmtProcessKillConnectionMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; - if (mgmtCheckRedirect(pMsg->thandle)) return; + if (mpeerCheckRedirect(pMsg->thandle)) return; SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index 880c6d0c10..eb20225e95 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -27,7 +27,7 @@ #include "mgmtDb.h" #include "tcluster.h" #include "tgrant.h" -#include "mgmtMnode.h" +#include "mpeer.h" #include "mgmtProfile.h" #include "mgmtSdb.h" #include "mgmtShell.h" @@ -142,14 +142,14 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { return; } - if (mgmtCheckRedirect(rpcMsg->handle)) { + if (mpeerCheckRedirect(rpcMsg->handle)) { // rpcSendRedirectRsp(rpcMsg->handle, mgmtGetMnodeIpListForRedirect()); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NO_MASTER); rpcFreeCont(rpcMsg->pCont); return; } - if (!mgmtInServerStatus()) { + if (!mpeerInServerStatus()) { mgmtProcessMsgWhileNotReady(rpcMsg); rpcFreeCont(rpcMsg->pCont); return; @@ -337,9 +337,9 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) { } if (pMsg->usePublicIp) { - mgmtGetMnodePublicIpList(&pHBRsp->ipList); + mpeerGetPublicIpList(&pHBRsp->ipList); } else { - mgmtGetMnodePrivateIpList(&pHBRsp->ipList); + mpeerGetPrivateIpList(&pHBRsp->ipList); } /* @@ -423,9 +423,9 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) { pConnectRsp->superAuth = pUser->superAuth; if (pMsg->usePublicIp) { - mgmtGetMnodePublicIpList(&pConnectRsp->ipList); + mpeerGetPublicIpList(&pConnectRsp->ipList); } else { - mgmtGetMnodePrivateIpList(&pConnectRsp->ipList); + mpeerGetPrivateIpList(&pConnectRsp->ipList); } connect_over: diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index b536bb5ac9..8dfe835f32 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -35,7 +35,7 @@ #include "tcluster.h" #include "mgmtDServer.h" #include "tgrant.h" -#include "mgmtMnode.h" +#include "mpeer.h" #include "mgmtProfile.h" #include "mgmtSdb.h" #include "mgmtShell.h" @@ -1650,8 +1650,6 @@ static SChildTableObj* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_ } static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) { - if (mgmtCheckRedirect(rpcMsg->handle)) return; - SDMConfigTableMsg *pCfg = (SDMConfigTableMsg *) rpcMsg->pCont; pCfg->dnode = htonl(pCfg->dnode); pCfg->vnode = htonl(pCfg->vnode); diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index fe7d40e120..8e969b3c9c 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -20,7 +20,7 @@ #include "tutil.h" #include "taccount.h" #include "tgrant.h" -#include "mgmtMnode.h" +#include "mpeer.h" #include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtUser.h" @@ -314,8 +314,6 @@ SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp) { } static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg) { - if (mgmtCheckRedirect(pMsg->thandle)) return; - int32_t code; SUserObj *pUser = pMsg->pUser; @@ -333,8 +331,6 @@ static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg) { } static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { - if (mgmtCheckRedirect(pMsg->thandle)) return; - int32_t code; SUserObj *pOperUser = pMsg->pUser; @@ -427,8 +423,6 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { } static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) { - if (mgmtCheckRedirect(pMsg->thandle)) return; - int32_t code; SUserObj *pOperUser = pMsg->pUser; diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 6b27fbbc83..98ae422904 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -24,7 +24,7 @@ #include "mgmtDb.h" #include "mgmtDClient.h" #include "mgmtDServer.h" -#include "mgmtMnode.h" +#include "mpeer.h" #include "mgmtProfile.h" #include "mgmtSdb.h" #include "mgmtShell.h" @@ -663,8 +663,6 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) { } static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) { - if (mgmtCheckRedirect(rpcMsg->handle)) return; - SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) rpcMsg->pCont; pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->vgId = htonl(pCfg->vgId); From b9c4fc22654f6d7474348ed93f6a885a05b50459 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 11 Apr 2020 00:42:21 +0800 Subject: [PATCH 41/78] change go driver to git submodule. use following commands to retreive: git clone https://github.com/taosdata/TDengine git checkout develop git pull git submodule update --init --recursive --- .gitmodules | 3 + src/connector/go | 1 + src/connector/go/src/taosSql/connection.go | 368 ------------------ src/connector/go/src/taosSql/connector.go | 49 --- src/connector/go/src/taosSql/const.go | 51 --- src/connector/go/src/taosSql/driver.go | 43 --- src/connector/go/src/taosSql/dsn.go | 202 ---------- src/connector/go/src/taosSql/result.go | 200 ---------- src/connector/go/src/taosSql/rows.go | 296 --------------- src/connector/go/src/taosSql/statement.go | 158 -------- src/connector/go/src/taosSql/taosLog.go | 120 ------ src/connector/go/src/taosSql/taosSqlCgo.go | 84 ---- src/connector/go/src/taosSql/utils.go | 422 --------------------- 13 files changed, 4 insertions(+), 1993 deletions(-) create mode 100644 .gitmodules create mode 160000 src/connector/go delete mode 100755 src/connector/go/src/taosSql/connection.go delete mode 100755 src/connector/go/src/taosSql/connector.go delete mode 100755 src/connector/go/src/taosSql/const.go delete mode 100755 src/connector/go/src/taosSql/driver.go delete mode 100755 src/connector/go/src/taosSql/dsn.go delete mode 100755 src/connector/go/src/taosSql/result.go delete mode 100755 src/connector/go/src/taosSql/rows.go delete mode 100755 src/connector/go/src/taosSql/statement.go delete mode 100755 src/connector/go/src/taosSql/taosLog.go delete mode 100755 src/connector/go/src/taosSql/taosSqlCgo.go delete mode 100755 src/connector/go/src/taosSql/utils.go diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..35aea3658b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "src/connector/go"] + path = src/connector/go + url = https://github.com/taosdata/driver-go diff --git a/src/connector/go b/src/connector/go new file mode 160000 index 0000000000..8c58c512b6 --- /dev/null +++ b/src/connector/go @@ -0,0 +1 @@ +Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766 diff --git a/src/connector/go/src/taosSql/connection.go b/src/connector/go/src/taosSql/connection.go deleted file mode 100755 index 2a3f5a9acd..0000000000 --- a/src/connector/go/src/taosSql/connection.go +++ /dev/null @@ -1,368 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package taosSql - -import "C" -import ( - "context" - "errors" - "database/sql/driver" - "unsafe" - "strconv" - "strings" - "time" -) - -type taosConn struct { - taos unsafe.Pointer - affectedRows int - insertId int - cfg *config - status statusFlag - parseTime bool - reset bool // set when the Go SQL package calls ResetSession -} - -type taosSqlResult struct { - affectedRows int64 - insertId int64 -} - -func (res *taosSqlResult) LastInsertId() (int64, error) { - return res.insertId, nil -} - -func (res *taosSqlResult) RowsAffected() (int64, error) { - return res.affectedRows, nil -} - -func (mc *taosConn) Begin() (driver.Tx, error) { - taosLog.Println("taosSql not support transaction") - return nil, errors.New("taosSql not support transaction") -} - -func (mc *taosConn) Close() (err error) { - if mc.taos == nil { - return errConnNoExist - } - mc.taos_close() - return nil -} - -func (mc *taosConn) Prepare(query string) (driver.Stmt, error) { - if mc.taos == nil { - return nil, errInvalidConn - } - - stmt := &taosSqlStmt{ - mc: mc, - pSql: query, - } - - // find ? count and save to stmt.paramCount - stmt.paramCount = strings.Count(query, "?") - - //fmt.Printf("prepare alloc stmt:%p, sql:%s\n", stmt, query) - taosLog.Printf("prepare alloc stmt:%p, sql:%s\n", stmt, query) - - return stmt, nil -} - -func (mc *taosConn) interpolateParams(query string, args []driver.Value) (string, error) { - // Number of ? should be same to len(args) - if strings.Count(query, "?") != len(args) { - return "", driver.ErrSkip - } - - buf := make([]byte, defaultBufSize) - buf = buf[:0] // clear buf - argPos := 0 - - for i := 0; i < len(query); i++ { - q := strings.IndexByte(query[i:], '?') - if q == -1 { - buf = append(buf, query[i:]...) - break - } - buf = append(buf, query[i:i+q]...) - i += q - - arg := args[argPos] - argPos++ - - if arg == nil { - buf = append(buf, "NULL"...) - continue - } - - switch v := arg.(type) { - case int64: - buf = strconv.AppendInt(buf, v, 10) - case uint64: - // Handle uint64 explicitly because our custom ConvertValue emits unsigned values - buf = strconv.AppendUint(buf, v, 10) - case float64: - buf = strconv.AppendFloat(buf, v, 'g', -1, 64) - case bool: - if v { - buf = append(buf, '1') - } else { - buf = append(buf, '0') - } - case time.Time: - if v.IsZero() { - buf = append(buf, "'0000-00-00'"...) - } else { - v := v.In(mc.cfg.loc) - v = v.Add(time.Nanosecond * 500) // To round under microsecond - year := v.Year() - year100 := year / 100 - year1 := year % 100 - month := v.Month() - day := v.Day() - hour := v.Hour() - minute := v.Minute() - second := v.Second() - micro := v.Nanosecond() / 1000 - - buf = append(buf, []byte{ - '\'', - digits10[year100], digits01[year100], - digits10[year1], digits01[year1], - '-', - digits10[month], digits01[month], - '-', - digits10[day], digits01[day], - ' ', - digits10[hour], digits01[hour], - ':', - digits10[minute], digits01[minute], - ':', - digits10[second], digits01[second], - }...) - - if micro != 0 { - micro10000 := micro / 10000 - micro100 := micro / 100 % 100 - micro1 := micro % 100 - buf = append(buf, []byte{ - '.', - digits10[micro10000], digits01[micro10000], - digits10[micro100], digits01[micro100], - digits10[micro1], digits01[micro1], - }...) - } - buf = append(buf, '\'') - } - case []byte: - if v == nil { - buf = append(buf, "NULL"...) - } else { - buf = append(buf, "_binary'"...) - if mc.status&statusNoBackslashEscapes == 0 { - buf = escapeBytesBackslash(buf, v) - } else { - buf = escapeBytesQuotes(buf, v) - } - buf = append(buf, '\'') - } - case string: - //buf = append(buf, '\'') - if mc.status&statusNoBackslashEscapes == 0 { - buf = escapeStringBackslash(buf, v) - } else { - buf = escapeStringQuotes(buf, v) - } - //buf = append(buf, '\'') - default: - return "", driver.ErrSkip - } - - //if len(buf)+4 > mc.maxAllowedPacket { - if len(buf)+4 > maxTaosSqlLen { - return "", driver.ErrSkip - } - } - if argPos != len(args) { - return "", driver.ErrSkip - } - return string(buf), nil -} - -func (mc *taosConn) Exec(query string, args []driver.Value) (driver.Result, error) { - if mc.taos == nil { - return nil, driver.ErrBadConn - } - if len(args) != 0 { - if !mc.cfg.interpolateParams { - return nil, driver.ErrSkip - } - // try to interpolate the parameters to save extra roundtrips for preparing and closing a statement - prepared, err := mc.interpolateParams(query, args) - if err != nil { - return nil, err - } - query = prepared - } - - mc.affectedRows = 0 - mc.insertId = 0 - _, err := mc.taosQuery(query) - if err == nil { - return &taosSqlResult{ - affectedRows: int64(mc.affectedRows), - insertId: int64(mc.insertId), - }, err - } - - return nil, err -} - -func (mc *taosConn) Query(query string, args []driver.Value) (driver.Rows, error) { - return mc.query(query, args) -} - -func (mc *taosConn) query(query string, args []driver.Value) (*textRows, error) { - if mc.taos == nil { - return nil, driver.ErrBadConn - } - if len(args) != 0 { - if !mc.cfg.interpolateParams { - return nil, driver.ErrSkip - } - // try client-side prepare to reduce roundtrip - prepared, err := mc.interpolateParams(query, args) - if err != nil { - return nil, err - } - query = prepared - } - - num_fields, err := mc.taosQuery(query) - if err == nil { - // Read Result - rows := new(textRows) - rows.mc = mc - - // Columns field - rows.rs.columns, err = mc.readColumns(num_fields) - return rows, err - } - return nil, err -} - -// Ping implements driver.Pinger interface -func (mc *taosConn) Ping(ctx context.Context) (err error) { - if mc.taos != nil { - return nil - } - return errInvalidConn -} - -// BeginTx implements driver.ConnBeginTx interface -func (mc *taosConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { - taosLog.Println("taosSql not support transaction") - return nil, errors.New("taosSql not support transaction") -} - -func (mc *taosConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { - if mc.taos == nil { - return nil, errInvalidConn - } - - dargs, err := namedValueToValue(args) - if err != nil { - return nil, err - } - - rows, err := mc.query(query, dargs) - if err != nil { - return nil, err - } - - return rows, err -} - -func (mc *taosConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { - if mc.taos == nil { - return nil, errInvalidConn - } - - dargs, err := namedValueToValue(args) - if err != nil { - return nil, err - } - - return mc.Exec(query, dargs) -} - -func (mc *taosConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { - if mc.taos == nil { - return nil, errInvalidConn - } - - stmt, err := mc.Prepare(query) - if err != nil { - return nil, err - } - - return stmt, nil -} - -func (stmt *taosSqlStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { - if stmt.mc == nil { - return nil, errInvalidConn - } - dargs, err := namedValueToValue(args) - - if err != nil { - return nil, err - } - - rows, err := stmt.query(dargs) - if err != nil { - return nil, err - } - return rows, err -} - -func (stmt *taosSqlStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { - if stmt.mc == nil { - return nil, errInvalidConn - } - - dargs, err := namedValueToValue(args) - if err != nil { - return nil, err - } - - return stmt.Exec(dargs) -} - -func (mc *taosConn) CheckNamedValue(nv *driver.NamedValue) (err error) { - nv.Value, err = converter{}.ConvertValue(nv.Value) - return -} - -// ResetSession implements driver.SessionResetter. -// (From Go 1.10) -func (mc *taosConn) ResetSession(ctx context.Context) error { - if mc.taos == nil { - return driver.ErrBadConn - } - mc.reset = true - return nil -} diff --git a/src/connector/go/src/taosSql/connector.go b/src/connector/go/src/taosSql/connector.go deleted file mode 100755 index 55715b949e..0000000000 --- a/src/connector/go/src/taosSql/connector.go +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -import ( - "context" - "database/sql/driver" -) - -type connector struct { - cfg *config -} - -// Connect implements driver.Connector interface. -// Connect returns a connection to the database. -func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { - var err error - // New taosConn - mc := &taosConn{ - cfg: c.cfg, - parseTime: c.cfg.parseTime, - } - - // Connect to Server - mc.taos, err = mc.taosConnect(mc.cfg.addr, mc.cfg.user, mc.cfg.passwd, mc.cfg.dbName, mc.cfg.port) - if err != nil { - return nil, err - } - - return mc, nil -} - -// Driver implements driver.Connector interface. -// Driver returns &taosSQLDriver{}. -func (c *connector) Driver() driver.Driver { - return &taosSQLDriver{} -} diff --git a/src/connector/go/src/taosSql/const.go b/src/connector/go/src/taosSql/const.go deleted file mode 100755 index 89214a0d45..0000000000 --- a/src/connector/go/src/taosSql/const.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package taosSql - -const ( - timeFormat = "2006-01-02 15:04:05" - maxTaosSqlLen = 65380 - defaultBufSize = maxTaosSqlLen + 32 -) - -type fieldType byte - -type fieldFlag uint16 - -const ( - flagNotNULL fieldFlag = 1 << iota -) - -type statusFlag uint16 - -const ( - statusInTrans statusFlag = 1 << iota - statusInAutocommit - statusReserved // Not in documentation - statusMoreResultsExists - statusNoGoodIndexUsed - statusNoIndexUsed - statusCursorExists - statusLastRowSent - statusDbDropped - statusNoBackslashEscapes - statusMetadataChanged - statusQueryWasSlow - statusPsOutParams - statusInTransReadonly - statusSessionStateChanged -) - diff --git a/src/connector/go/src/taosSql/driver.go b/src/connector/go/src/taosSql/driver.go deleted file mode 100755 index 6839979c61..0000000000 --- a/src/connector/go/src/taosSql/driver.go +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -import ( - "context" - "database/sql" - "database/sql/driver" -) - -// taosSqlDriver is exported to make the driver directly accessible. -// In general the driver is used via the database/sql package. -type taosSQLDriver struct{} - -// Open new Connection. -// the DSN string is formatted -func (d taosSQLDriver) Open(dsn string) (driver.Conn, error) { - cfg, err := parseDSN(dsn) - if err != nil { - return nil, err - } - c := &connector{ - cfg: cfg, - } - return c.Connect(context.Background()) -} - -func init() { - sql.Register("taosSql", &taosSQLDriver{}) - taosLogInit() -} diff --git a/src/connector/go/src/taosSql/dsn.go b/src/connector/go/src/taosSql/dsn.go deleted file mode 100755 index c68ae1725c..0000000000 --- a/src/connector/go/src/taosSql/dsn.go +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package taosSql - -import ( - "errors" - "net/url" - "strconv" - "strings" - "time" -) - -var ( - errInvalidDSNUnescaped = errors.New("invalid DSN: did you forget to escape a param value?") - errInvalidDSNAddr = errors.New("invalid DSN: network address not terminated (missing closing brace)") - errInvalidDSNPort = errors.New("invalid DSN: network port is not a valid number") - errInvalidDSNNoSlash = errors.New("invalid DSN: missing the slash separating the database name") -) - -// Config is a configuration parsed from a DSN string. -// If a new Config is created instead of being parsed from a DSN string, -// the NewConfig function should be used, which sets default values. -type config struct { - user string // Username - passwd string // Password (requires User) - net string // Network type - addr string // Network address (requires Net) - port int - dbName string // Database name - params map[string]string // Connection parameters - loc *time.Location // Location for time.Time values - columnsWithAlias bool // Prepend table alias to column names - interpolateParams bool // Interpolate placeholders into query string - parseTime bool // Parse time values to time.Time -} - -// NewConfig creates a new Config and sets default values. -func newConfig() *config { - return &config{ - loc: time.UTC, - interpolateParams: true, - parseTime: true, - } -} - -// ParseDSN parses the DSN string to a Config -func parseDSN(dsn string) (cfg *config, err error) { - taosLog.Println("input dsn:", dsn) - - // New config with some default values - cfg = newConfig() - - // [user[:password]@][net[(addr)]]/dbname[?param1=value1¶mN=valueN] - // Find the last '/' (since the password or the net addr might contain a '/') - foundSlash := false - for i := len(dsn) - 1; i >= 0; i-- { - if dsn[i] == '/' { - foundSlash = true - var j, k int - - // left part is empty if i <= 0 - if i > 0 { - // [username[:password]@][protocol[(address)]] - // Find the last '@' in dsn[:i] - for j = i; j >= 0; j-- { - if dsn[j] == '@' { - // username[:password] - // Find the first ':' in dsn[:j] - for k = 0; k < j; k++ { - if dsn[k] == ':' { - cfg.passwd = dsn[k+1 : j] - break - } - } - cfg.user = dsn[:k] - - break - } - } - - // [protocol[(address)]] - // Find the first '(' in dsn[j+1:i] - for k = j + 1; k < i; k++ { - if dsn[k] == '(' { - // dsn[i-1] must be == ')' if an address is specified - if dsn[i-1] != ')' { - if strings.ContainsRune(dsn[k+1:i], ')') { - return nil, errInvalidDSNUnescaped - } - return nil, errInvalidDSNAddr - } - strs := strings.Split(dsn[k+1:i-1], ":") - if len(strs) == 1 { - return nil, errInvalidDSNAddr - } - cfg.addr = strs[0] - cfg.port, err = strconv.Atoi(strs[1]) - if err != nil { - return nil, errInvalidDSNPort - } - break - } - } - cfg.net = dsn[j+1 : k] - } - - // dbname[?param1=value1&...¶mN=valueN] - // Find the first '?' in dsn[i+1:] - for j = i + 1; j < len(dsn); j++ { - if dsn[j] == '?' { - if err = parseDSNParams(cfg, dsn[j+1:]); err != nil { - return - } - break - } - } - cfg.dbName = dsn[i+1 : j] - - break - } - } - - if !foundSlash && len(dsn) > 0 { - return nil, errInvalidDSNNoSlash - } - - taosLog.Printf("cfg info: %+v", cfg) - - return -} - -// parseDSNParams parses the DSN "query string" -// Values must be url.QueryEscape'ed -func parseDSNParams(cfg *config, params string) (err error) { - for _, v := range strings.Split(params, "&") { - param := strings.SplitN(v, "=", 2) - if len(param) != 2 { - continue - } - - // cfg params - switch value := param[1]; param[0] { - case "columnsWithAlias": - var isBool bool - cfg.columnsWithAlias, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - // Enable client side placeholder substitution - case "interpolateParams": - var isBool bool - cfg.interpolateParams, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - // Time Location - case "loc": - if value, err = url.QueryUnescape(value); err != nil { - return - } - cfg.loc, err = time.LoadLocation(value) - if err != nil { - return - } - - // time.Time parsing - case "parseTime": - var isBool bool - cfg.parseTime, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - default: - // lazy init - if cfg.params == nil { - cfg.params = make(map[string]string) - } - - if cfg.params[param[0]], err = url.QueryUnescape(value); err != nil { - return - } - } - } - - return -} diff --git a/src/connector/go/src/taosSql/result.go b/src/connector/go/src/taosSql/result.go deleted file mode 100755 index 2367560c31..0000000000 --- a/src/connector/go/src/taosSql/result.go +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -/* -#cgo CFLAGS : -I/usr/include -#cgo LDFLAGS: -L/usr/lib -ltaos -#include -#include -#include -#include -*/ -import "C" - -import ( - "database/sql/driver" - "errors" - "fmt" - "io" - "strconv" - "time" - "unsafe" -) - -/****************************************************************************** -* Result * -******************************************************************************/ -// Read Packets as Field Packets until EOF-Packet or an Error appears -func (mc *taosConn) readColumns(count int) ([]taosSqlField, error) { - - columns := make([]taosSqlField, count) - var result unsafe.Pointer - result = C.taos_use_result(mc.taos) - if result == nil { - return nil, errors.New("invalid result") - } - - pFields := (*C.struct_taosField)(C.taos_fetch_fields(result)) - - // TODO: Optimized rewriting !!!! - fields := (*[1 << 30]C.struct_taosField)(unsafe.Pointer(pFields)) - - for i := 0; i < count; i++ { - //columns[i].tableName = ms.taos. - //fmt.Println(reflect.TypeOf(fields[i].name)) - var charray []byte - for j := range fields[i].name { - //fmt.Println("fields[i].name[j]: ", fields[i].name[j]) - if fields[i].name[j] != 0 { - charray = append(charray, byte(fields[i].name[j])) - } else { - break - } - } - columns[i].name = string(charray) - columns[i].length = (uint32)(fields[i].bytes) - columns[i].fieldType = fieldType(fields[i]._type) - columns[i].flags = 0 - // columns[i].decimals = 0 - //columns[i].charSet = 0 - } - return columns, nil -} - -func (rows *taosSqlRows) readRow(dest []driver.Value) error { - mc := rows.mc - - if rows.rs.done || mc == nil { - return io.EOF - } - - var result unsafe.Pointer - result = C.taos_use_result(mc.taos) - if result == nil { - return errors.New(C.GoString(C.taos_errstr(mc.taos))) - } - - //var row *unsafe.Pointer - row := C.taos_fetch_row(result) - if row == nil { - rows.rs.done = true - C.taos_free_result(result) - rows.mc = nil - return io.EOF - } - - // because sizeof(void*) == sizeof(int*) == 8 - // notes: sizeof(int) == 8 in go, but sizeof(int) == 4 in C. - for i := range dest { - currentRow := (unsafe.Pointer)(uintptr(*((*int)(unsafe.Pointer(uintptr(unsafe.Pointer(row)) + uintptr(i)*unsafe.Sizeof(int(0))))))) - - if currentRow == nil { - dest[i] = nil - continue - } - - switch rows.rs.columns[i].fieldType { - case C.TSDB_DATA_TYPE_BOOL: - if (*((*byte)(currentRow))) != 0 { - dest[i] = true - } else { - dest[i] = false - } - break - - case C.TSDB_DATA_TYPE_TINYINT: - dest[i] = (int)(*((*byte)(currentRow))) - break - - case C.TSDB_DATA_TYPE_SMALLINT: - dest[i] = (int16)(*((*int16)(currentRow))) - break - - case C.TSDB_DATA_TYPE_INT: - dest[i] = (int)(*((*int32)(currentRow))) // notes int32 of go <----> int of C - break - - case C.TSDB_DATA_TYPE_BIGINT: - dest[i] = (int64)(*((*int64)(currentRow))) - break - - case C.TSDB_DATA_TYPE_FLOAT: - dest[i] = (*((*float32)(currentRow))) - break - - case C.TSDB_DATA_TYPE_DOUBLE: - dest[i] = (*((*float64)(currentRow))) - break - - case C.TSDB_DATA_TYPE_BINARY, C.TSDB_DATA_TYPE_NCHAR: - charLen := rows.rs.columns[i].length - var index uint32 - binaryVal := make([]byte, charLen) - for index = 0; index < charLen; index++ { - binaryVal[index] = *((*byte)(unsafe.Pointer(uintptr(currentRow) + uintptr(index)))) - } - dest[i] = string(binaryVal[:]) - break - - case C.TSDB_DATA_TYPE_TIMESTAMP: - if mc.cfg.parseTime == true { - timestamp := (int64)(*((*int64)(currentRow))) - dest[i] = timestampConvertToString(timestamp, int(C.taos_result_precision(result))) - } else { - dest[i] = (int64)(*((*int64)(currentRow))) - } - break - - default: - fmt.Println("default fieldType: set dest[] to nil") - dest[i] = nil - break - } - } - - return nil -} - -// Read result as Field format until all rows or an Error appears -// call this func in conn mode -func (rows *textRows) readRow(dest []driver.Value) error { - return rows.taosSqlRows.readRow(dest) -} - -// call thsi func in stmt mode -func (rows *binaryRows) readRow(dest []driver.Value) error { - return rows.taosSqlRows.readRow(dest) -} - -func timestampConvertToString(timestamp int64, precision int) string { - var decimal, sVal, nsVal int64 - if precision == 0 { - decimal = timestamp % 1000 - sVal = timestamp / 1000 - nsVal = decimal * 1000 - } else { - decimal = timestamp % 1000000 - sVal = timestamp / 1000000 - nsVal = decimal * 1000000 - } - - date_time := time.Unix(sVal, nsVal) - - //const base_format = "2006-01-02 15:04:05" - str_time := date_time.Format(timeFormat) - - return (str_time + "." + strconv.Itoa(int(decimal))) -} diff --git a/src/connector/go/src/taosSql/rows.go b/src/connector/go/src/taosSql/rows.go deleted file mode 100755 index 5040dca06d..0000000000 --- a/src/connector/go/src/taosSql/rows.go +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql -/* -#cgo CFLAGS : -I/usr/include -#cgo LDFLAGS: -L/usr/lib -ltaos -#include -#include -#include -#include -*/ -import "C" - -import ( - "database/sql" - "database/sql/driver" - "io" - "math" - "reflect" -) - -type taosSqlField struct { - tableName string - name string - length uint32 - flags fieldFlag // indicate whether this field can is null - fieldType fieldType - decimals byte - charSet uint8 -} - -type resultSet struct { - columns []taosSqlField - columnNames []string - done bool -} - -type taosSqlRows struct { - mc *taosConn - rs resultSet -} - -type binaryRows struct { - taosSqlRows -} - -type textRows struct { - taosSqlRows -} - -func (rows *taosSqlRows) Columns() []string { - if rows.rs.columnNames != nil { - return rows.rs.columnNames - } - - columns := make([]string, len(rows.rs.columns)) - if rows.mc != nil && rows.mc.cfg.columnsWithAlias { - for i := range columns { - if tableName := rows.rs.columns[i].tableName; len(tableName) > 0 { - columns[i] = tableName + "." + rows.rs.columns[i].name - } else { - columns[i] = rows.rs.columns[i].name - } - } - } else { - for i := range columns { - columns[i] = rows.rs.columns[i].name - } - } - - rows.rs.columnNames = columns - - return columns -} - -func (rows *taosSqlRows) ColumnTypeDatabaseTypeName(i int) string { - return rows.rs.columns[i].typeDatabaseName() -} - -func (rows *taosSqlRows) ColumnTypeLength(i int) (length int64, ok bool) { - return int64(rows.rs.columns[i].length), true -} - -func (rows *taosSqlRows) ColumnTypeNullable(i int) (nullable, ok bool) { - return rows.rs.columns[i].flags&flagNotNULL == 0, true -} - -func (rows *taosSqlRows) ColumnTypePrecisionScale(i int) (int64, int64, bool) { - column := rows.rs.columns[i] - decimals := int64(column.decimals) - - switch column.fieldType { - case C.TSDB_DATA_TYPE_FLOAT: - fallthrough - case C.TSDB_DATA_TYPE_DOUBLE: - if decimals == 0x1f { - return math.MaxInt64, math.MaxInt64, true - } - return math.MaxInt64, decimals, true - } - - return 0, 0, false -} - -func (rows *taosSqlRows) ColumnTypeScanType(i int) reflect.Type { - return rows.rs.columns[i].scanType() -} - -func (rows *taosSqlRows) Close() error { - if rows.mc != nil { - result := C.taos_use_result(rows.mc.taos) - if result != nil { - C.taos_free_result(result) - } - rows.mc = nil - } - return nil -} - -func (rows *taosSqlRows) HasNextResultSet() (b bool) { - if rows.mc == nil { - return false - } - return rows.mc.status&statusMoreResultsExists != 0 -} - -func (rows *taosSqlRows) nextResultSet() (int, error) { - if rows.mc == nil { - return 0, io.EOF - } - - // Remove unread packets from stream - if !rows.rs.done { - rows.rs.done = true - } - - if !rows.HasNextResultSet() { - rows.mc = nil - return 0, io.EOF - } - rows.rs = resultSet{} - return 0,nil -} - -func (rows *taosSqlRows) nextNotEmptyResultSet() (int, error) { - for { - resLen, err := rows.nextResultSet() - if err != nil { - return 0, err - } - - if resLen > 0 { - return resLen, nil - } - - rows.rs.done = true - } -} - -func (rows *binaryRows) NextResultSet() error { - resLen, err := rows.nextNotEmptyResultSet() - if err != nil { - return err - } - - rows.rs.columns, err = rows.mc.readColumns(resLen) - return err -} - -// stmt.Query return binary rows, and get row from this func -func (rows *binaryRows) Next(dest []driver.Value) error { - if mc := rows.mc; mc != nil { - // Fetch next row from stream - return rows.readRow(dest) - } - return io.EOF -} - -func (rows *textRows) NextResultSet() (err error) { - resLen, err := rows.nextNotEmptyResultSet() - if err != nil { - return err - } - - rows.rs.columns, err = rows.mc.readColumns(resLen) - return err -} - -// db.Query return text rows, and get row from this func -func (rows *textRows) Next(dest []driver.Value) error { - if mc := rows.mc; mc != nil { - // Fetch next row from stream - return rows.readRow(dest) - } - return io.EOF -} - -func (mf *taosSqlField) typeDatabaseName() string { - //fmt.Println("######## (mf *taosSqlField) typeDatabaseName() mf.fieldType:", mf.fieldType) - switch mf.fieldType { - case C.TSDB_DATA_TYPE_BOOL: - return "BOOL" - - case C.TSDB_DATA_TYPE_TINYINT: - return "TINYINT" - - case C.TSDB_DATA_TYPE_SMALLINT: - return "SMALLINT" - - case C.TSDB_DATA_TYPE_INT: - return "INT" - - case C.TSDB_DATA_TYPE_BIGINT: - return "BIGINT" - - case C.TSDB_DATA_TYPE_FLOAT: - return "FLOAT" - - case C.TSDB_DATA_TYPE_DOUBLE: - return "DOUBLE" - - case C.TSDB_DATA_TYPE_BINARY: - return "BINARY" - - case C.TSDB_DATA_TYPE_NCHAR: - return "NCHAR" - - case C.TSDB_DATA_TYPE_TIMESTAMP: - return "TIMESTAMP" - - default: - return "" - } -} - -var ( - scanTypeFloat32 = reflect.TypeOf(float32(0)) - scanTypeFloat64 = reflect.TypeOf(float64(0)) - scanTypeInt8 = reflect.TypeOf(int8(0)) - scanTypeInt16 = reflect.TypeOf(int16(0)) - scanTypeInt32 = reflect.TypeOf(int32(0)) - scanTypeInt64 = reflect.TypeOf(int64(0)) - scanTypeNullTime = reflect.TypeOf(NullTime{}) - scanTypeRawBytes = reflect.TypeOf(sql.RawBytes{}) - scanTypeUnknown = reflect.TypeOf(new(interface{})) -) - -func (mf *taosSqlField) scanType() reflect.Type { - //fmt.Println("######## (mf *taosSqlField) scanType() mf.fieldType:", mf.fieldType) - switch mf.fieldType { - case C.TSDB_DATA_TYPE_BOOL: - return scanTypeInt8 - - case C.TSDB_DATA_TYPE_TINYINT: - return scanTypeInt8 - - case C.TSDB_DATA_TYPE_SMALLINT: - return scanTypeInt16 - - case C.TSDB_DATA_TYPE_INT: - return scanTypeInt32 - - case C.TSDB_DATA_TYPE_BIGINT: - return scanTypeInt64 - - case C.TSDB_DATA_TYPE_FLOAT: - return scanTypeFloat32 - - case C.TSDB_DATA_TYPE_DOUBLE: - return scanTypeFloat64 - - case C.TSDB_DATA_TYPE_BINARY: - return scanTypeRawBytes - - case C.TSDB_DATA_TYPE_NCHAR: - return scanTypeRawBytes - - case C.TSDB_DATA_TYPE_TIMESTAMP: - return scanTypeNullTime - - default: - return scanTypeUnknown - } -} diff --git a/src/connector/go/src/taosSql/statement.go b/src/connector/go/src/taosSql/statement.go deleted file mode 100755 index 3aa3e01564..0000000000 --- a/src/connector/go/src/taosSql/statement.go +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -import ( - "database/sql/driver" - "fmt" - "reflect" -) - -type taosSqlStmt struct { - mc *taosConn - id uint32 - pSql string - paramCount int -} - -func (stmt *taosSqlStmt) Close() error { - return nil -} - -func (stmt *taosSqlStmt) NumInput() int { - return stmt.paramCount -} - -func (stmt *taosSqlStmt) Exec(args []driver.Value) (driver.Result, error) { - if stmt.mc == nil || stmt.mc.taos == nil { - return nil, errInvalidConn - } - return stmt.mc.Exec(stmt.pSql, args) -} - -func (stmt *taosSqlStmt) Query(args []driver.Value) (driver.Rows, error) { - if stmt.mc == nil || stmt.mc.taos == nil { - return nil, errInvalidConn - } - return stmt.query(args) -} - -func (stmt *taosSqlStmt) query(args []driver.Value) (*binaryRows, error) { - mc := stmt.mc - if mc == nil || mc.taos == nil { - return nil, errInvalidConn - } - - querySql := stmt.pSql - - if len(args) != 0 { - if !mc.cfg.interpolateParams { - return nil, driver.ErrSkip - } - // try client-side prepare to reduce roundtrip - prepared, err := mc.interpolateParams(stmt.pSql, args) - if err != nil { - return nil, err - } - querySql = prepared - } - - num_fields, err := mc.taosQuery(querySql) - if err == nil { - // Read Result - rows := new(binaryRows) - rows.mc = mc - // Columns field - rows.rs.columns, err = mc.readColumns(num_fields) - return rows, err - } - return nil, err -} - -type converter struct{} - -// ConvertValue mirrors the reference/default converter in database/sql/driver -// with _one_ exception. We support uint64 with their high bit and the default -// implementation does not. This function should be kept in sync with -// database/sql/driver defaultConverter.ConvertValue() except for that -// deliberate difference. -func (c converter) ConvertValue(v interface{}) (driver.Value, error) { - - if driver.IsValue(v) { - return v, nil - } - - if vr, ok := v.(driver.Valuer); ok { - sv, err := callValuerValue(vr) - if err != nil { - return nil, err - } - if !driver.IsValue(sv) { - return nil, fmt.Errorf("non-Value type %T returned from Value", sv) - } - - return sv, nil - } - - rv := reflect.ValueOf(v) - switch rv.Kind() { - case reflect.Ptr: - // indirect pointers - if rv.IsNil() { - return nil, nil - } else { - return c.ConvertValue(rv.Elem().Interface()) - } - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return rv.Int(), nil - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - return rv.Uint(), nil - case reflect.Float32, reflect.Float64: - return rv.Float(), nil - case reflect.Bool: - return rv.Bool(), nil - case reflect.Slice: - ek := rv.Type().Elem().Kind() - if ek == reflect.Uint8 { - return rv.Bytes(), nil - } - return nil, fmt.Errorf("unsupported type %T, a slice of %s", v, ek) - case reflect.String: - return rv.String(), nil - } - return nil, fmt.Errorf("unsupported type %T, a %s", v, rv.Kind()) -} - -var valuerReflectType = reflect.TypeOf((*driver.Valuer)(nil)).Elem() - -// callValuerValue returns vr.Value(), with one exception: -// If vr.Value is an auto-generated method on a pointer type and the -// pointer is nil, it would panic at runtime in the panicwrap -// method. Treat it like nil instead. -// -// This is so people can implement driver.Value on value types and -// still use nil pointers to those types to mean nil/NULL, just like -// string/*string. -// -// This is an exact copy of the same-named unexported function from the -// database/sql package. -func callValuerValue(vr driver.Valuer) (v driver.Value, err error) { - if rv := reflect.ValueOf(vr); rv.Kind() == reflect.Ptr && - rv.IsNil() && - rv.Type().Elem().Implements(valuerReflectType) { - return nil, nil - } - return vr.Value() -} diff --git a/src/connector/go/src/taosSql/taosLog.go b/src/connector/go/src/taosSql/taosLog.go deleted file mode 100755 index 63af5e34c1..0000000000 --- a/src/connector/go/src/taosSql/taosLog.go +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package taosSql - -import ( - "bufio" - "errors" - "fmt" - "io" - "log" - "os" - "strings" -) - -// Various errors the driver might return. -var ( - errInvalidConn = errors.New("invalid connection") - errConnNoExist = errors.New("no existent connection ") -) - -var taosLog *log.Logger - -// SetLogger is used to set the logger for critical errors. -// The initial logger -func taosLogInit() { - cfgName := "/etc/taos/taos.cfg" - logNameDefault := "/var/log/taos/taosgo.log" - var logName string - - // get log path from cfg file - cfgFile, err := os.OpenFile(cfgName, os.O_RDONLY, 0644) - defer cfgFile.Close() - if err != nil { - fmt.Println(err) - logName = logNameDefault - } else { - logName, err = getLogNameFromCfg(cfgFile) - if err != nil { - fmt.Println(err) - logName = logNameDefault - } - } - - logFile, err := os.OpenFile(logName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - taosLog = log.New(logFile, "", log.LstdFlags) - taosLog.SetPrefix("TAOS DRIVER ") - taosLog.SetFlags(log.LstdFlags|log.Lshortfile) -} - -func getLogNameFromCfg(f *os.File) (string, error) { - // Create file buf, *Reader - r := bufio.NewReader(f) - for { - //read one line, return to slice b - b, _, err := r.ReadLine() - if err != nil { - if err == io.EOF { - break - } - panic(err) - } - - // Remove space of left and right - s := strings.TrimSpace(string(b)) - if strings.Index(s, "#") == 0 { - // comment line - continue - } - - if len(s) == 0 { - continue - } - - var ns string - // If there is a comment on the right of the line, must be remove - index := strings.Index(s, "#") - if index > 0 { - // Gets the string to the left of the comment to determine whether it is empty - ns = s[:index] - if len(ns) == 0 { - continue - } - } else { - ns = s; - } - - ss := strings.Fields(ns) - if strings.Compare("logDir", ss[0]) != 0 { - continue - } - - if len(ss) < 2 { - break - } - - // Add a filename after the path - logName := ss[1] + "/taosgo.log" - return logName,nil - } - - return "", errors.New("no config log path, use default") -} - diff --git a/src/connector/go/src/taosSql/taosSqlCgo.go b/src/connector/go/src/taosSql/taosSqlCgo.go deleted file mode 100755 index cc3aaa1658..0000000000 --- a/src/connector/go/src/taosSql/taosSqlCgo.go +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -/* -#cgo CFLAGS : -I/usr/include -#cgo LDFLAGS: -L/usr/lib -ltaos -#include -#include -#include -#include -*/ -import "C" - -import ( - "errors" - "unsafe" -) - -func (mc *taosConn) taosConnect(ip, user, pass, db string, port int) (taos unsafe.Pointer, err error) { - cuser := C.CString(user) - cpass := C.CString(pass) - cip := C.CString(ip) - cdb := C.CString(db) - defer C.free(unsafe.Pointer(cip)) - defer C.free(unsafe.Pointer(cuser)) - defer C.free(unsafe.Pointer(cpass)) - defer C.free(unsafe.Pointer(cdb)) - - taosObj := C.taos_connect(cip, cuser, cpass, cdb, (C.ushort)(port)) - if taosObj == nil { - return nil, errors.New("taos_connect() fail!") - } - - return (unsafe.Pointer)(taosObj), nil -} - -func (mc *taosConn) taosQuery(sqlstr string) (int, error) { - //taosLog.Printf("taosQuery() input sql:%s\n", sqlstr) - - csqlstr := C.CString(sqlstr) - defer C.free(unsafe.Pointer(csqlstr)) - code := int(C.taos_query(mc.taos, csqlstr)) - - if 0 != code { - mc.taos_error() - errStr := C.GoString(C.taos_errstr(mc.taos)) - taosLog.Println("taos_query() failed:", errStr) - taosLog.Printf("taosQuery() input sql:%s\n", sqlstr) - return 0, errors.New(errStr) - } - - // read result and save into mc struct - num_fields := int(C.taos_field_count(mc.taos)) - if 0 == num_fields { // there are no select and show kinds of commands - mc.affectedRows = int(C.taos_affected_rows(mc.taos)) - mc.insertId = 0 - } - - return num_fields, nil -} - -func (mc *taosConn) taos_close() { - C.taos_close(mc.taos) -} - -func (mc *taosConn) taos_error() { - // free local resouce: allocated memory/metric-meta refcnt - //var pRes unsafe.Pointer - pRes := C.taos_use_result(mc.taos) - C.taos_free_result(pRes) -} diff --git a/src/connector/go/src/taosSql/utils.go b/src/connector/go/src/taosSql/utils.go deleted file mode 100755 index a104322fcc..0000000000 --- a/src/connector/go/src/taosSql/utils.go +++ /dev/null @@ -1,422 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package taosSql - -/* -#cgo CFLAGS : -I/usr/include -#include -#cgo LDFLAGS: -L/usr/lib -ltaos -void taosSetAllocMode(int mode, const char* path, _Bool autoDump); -void taosDumpMemoryLeak(); -*/ -import "C" - - -import ( - "database/sql/driver" - "errors" - "fmt" - "sync/atomic" - "time" - "unsafe" -) - -// Returns the bool value of the input. -// The 2nd return value indicates if the input was a valid bool value -func readBool(input string) (value bool, valid bool) { - switch input { - case "1", "true", "TRUE", "True": - return true, true - case "0", "false", "FALSE", "False": - return false, true - } - - // Not a valid bool value - return -} - -/****************************************************************************** -* Time related utils * -******************************************************************************/ - -// NullTime represents a time.Time that may be NULL. -// NullTime implements the Scanner interface so -// it can be used as a scan destination: -// -// var nt NullTime -// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) -// ... -// if nt.Valid { -// // use nt.Time -// } else { -// // NULL value -// } -// -// This NullTime implementation is not driver-specific -type NullTime struct { - Time time.Time - Valid bool // Valid is true if Time is not NULL -} - -// Scan implements the Scanner interface. -// The value type must be time.Time or string / []byte (formatted time-string), -// otherwise Scan fails. -func (nt *NullTime) Scan(value interface{}) (err error) { - if value == nil { - nt.Time, nt.Valid = time.Time{}, false - return - } - - switch v := value.(type) { - case time.Time: - nt.Time, nt.Valid = v, true - return - case []byte: - nt.Time, err = parseDateTime(string(v), time.UTC) - nt.Valid = (err == nil) - return - case string: - nt.Time, err = parseDateTime(v, time.UTC) - nt.Valid = (err == nil) - return - } - - nt.Valid = false - return fmt.Errorf("Can't convert %T to time.Time", value) -} - -// Value implements the driver Valuer interface. -func (nt NullTime) Value() (driver.Value, error) { - if !nt.Valid { - return nil, nil - } - return nt.Time, nil -} - -func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { - base := "0000-00-00 00:00:00.0000000" - switch len(str) { - case 10, 19, 21, 22, 23, 24, 25, 26: // up to "YYYY-MM-DD HH:MM:SS.MMMMMM" - if str == base[:len(str)] { - return - } - t, err = time.Parse(timeFormat[:len(str)], str) - default: - err = fmt.Errorf("invalid time string: %s", str) - return - } - - // Adjust location - if err == nil && loc != time.UTC { - y, mo, d := t.Date() - h, mi, s := t.Clock() - t, err = time.Date(y, mo, d, h, mi, s, t.Nanosecond(), loc), nil - } - - return -} - -// zeroDateTime is used in formatBinaryDateTime to avoid an allocation -// if the DATE or DATETIME has the zero value. -// It must never be changed. -// The current behavior depends on database/sql copying the result. -var zeroDateTime = []byte("0000-00-00 00:00:00.000000") - -const digits01 = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" -const digits10 = "0000000000111111111122222222223333333333444444444455555555556666666666777777777788888888889999999999" - -/****************************************************************************** -* Convert from and to bytes * -******************************************************************************/ - -func uint64ToBytes(n uint64) []byte { - return []byte{ - byte(n), - byte(n >> 8), - byte(n >> 16), - byte(n >> 24), - byte(n >> 32), - byte(n >> 40), - byte(n >> 48), - byte(n >> 56), - } -} - -func uint64ToString(n uint64) []byte { - var a [20]byte - i := 20 - - // U+0030 = 0 - // ... - // U+0039 = 9 - - var q uint64 - for n >= 10 { - i-- - q = n / 10 - a[i] = uint8(n-q*10) + 0x30 - n = q - } - - i-- - a[i] = uint8(n) + 0x30 - - return a[i:] -} - -// treats string value as unsigned integer representation -func stringToInt(b []byte) int { - val := 0 - for i := range b { - val *= 10 - val += int(b[i] - 0x30) - } - return val -} - -// reserveBuffer checks cap(buf) and expand buffer to len(buf) + appendSize. -// If cap(buf) is not enough, reallocate new buffer. -func reserveBuffer(buf []byte, appendSize int) []byte { - newSize := len(buf) + appendSize - if cap(buf) < newSize { - // Grow buffer exponentially - newBuf := make([]byte, len(buf)*2+appendSize) - copy(newBuf, buf) - buf = newBuf - } - return buf[:newSize] -} - -// escapeBytesBackslash escapes []byte with backslashes (\) -// This escapes the contents of a string (provided as []byte) by adding backslashes before special -// characters, and turning others into specific escape sequences, such as -// turning newlines into \n and null bytes into \0. -func escapeBytesBackslash(buf, v []byte) []byte { - pos := len(buf) - buf = reserveBuffer(buf, len(v)*2) - for _, c := range v { - switch c { - case '\x00': - buf[pos] = '\\' - buf[pos+1] = '0' - pos += 2 - case '\n': - buf[pos] = '\\' - buf[pos+1] = 'n' - pos += 2 - case '\r': - buf[pos] = '\\' - buf[pos+1] = 'r' - pos += 2 - case '\x1a': - buf[pos] = '\\' - buf[pos+1] = 'Z' - pos += 2 - case '\'': - buf[pos] = '\\' - buf[pos+1] = '\'' - pos += 2 - case '"': - buf[pos] = '\\' - buf[pos+1] = '"' - pos += 2 - case '\\': - buf[pos] = '\\' - buf[pos+1] = '\\' - pos += 2 - default: - buf[pos] = c - pos++ - } - } - return buf[:pos] -} - -// escapeStringBackslash is similar to escapeBytesBackslash but for string. -func escapeStringBackslash(buf []byte, v string) []byte { - pos := len(buf) - buf = reserveBuffer(buf, len(v)*2) - - for i := 0; i < len(v); i++ { - c := v[i] - switch c { - case '\x00': - buf[pos] = '\\' - buf[pos+1] = '0' - pos += 2 - case '\n': - buf[pos] = '\\' - buf[pos+1] = 'n' - pos += 2 - case '\r': - buf[pos] = '\\' - buf[pos+1] = 'r' - pos += 2 - case '\x1a': - buf[pos] = '\\' - buf[pos+1] = 'Z' - pos += 2 - //case '\'': - // buf[pos] = '\\' - // buf[pos+1] = '\'' - // pos += 2 - case '"': - buf[pos] = '\\' - buf[pos+1] = '"' - pos += 2 - case '\\': - buf[pos] = '\\' - buf[pos+1] = '\\' - pos += 2 - default: - buf[pos] = c - pos++ - } - } - - return buf[:pos] -} - -// escapeBytesQuotes escapes apostrophes in []byte by doubling them up. -// This escapes the contents of a string by doubling up any apostrophes that -// it contains. This is used when the NO_BACKSLASH_ESCAPES SQL_MODE is in -// effect on the server. -func escapeBytesQuotes(buf, v []byte) []byte { - pos := len(buf) - buf = reserveBuffer(buf, len(v)*2) - - for _, c := range v { - if c == '\'' { - buf[pos] = '\'' - buf[pos+1] = '\'' - pos += 2 - } else { - buf[pos] = c - pos++ - } - } - - return buf[:pos] -} - -// escapeStringQuotes is similar to escapeBytesQuotes but for string. -func escapeStringQuotes(buf []byte, v string) []byte { - pos := len(buf) - buf = reserveBuffer(buf, len(v)*2) - - for i := 0; i < len(v); i++ { - c := v[i] - if c == '\'' { - buf[pos] = '\'' - buf[pos+1] = '\'' - pos += 2 - } else { - buf[pos] = c - pos++ - } - } - - return buf[:pos] -} - -/****************************************************************************** -* Sync utils * -******************************************************************************/ - -// noCopy may be embedded into structs which must not be copied -// after the first use. -// -// See https://github.com/golang/go/issues/8005#issuecomment-190753527 -// for details. -type noCopy struct{} - -// Lock is a no-op used by -copylocks checker from `go vet`. -func (*noCopy) Lock() {} - -// atomicBool is a wrapper around uint32 for usage as a boolean value with -// atomic access. -type atomicBool struct { - _noCopy noCopy - value uint32 -} - -// IsSet returns whether the current boolean value is true -func (ab *atomicBool) IsSet() bool { - return atomic.LoadUint32(&ab.value) > 0 -} - -// Set sets the value of the bool regardless of the previous value -func (ab *atomicBool) Set(value bool) { - if value { - atomic.StoreUint32(&ab.value, 1) - } else { - atomic.StoreUint32(&ab.value, 0) - } -} - -// TrySet sets the value of the bool and returns whether the value changed -func (ab *atomicBool) TrySet(value bool) bool { - if value { - return atomic.SwapUint32(&ab.value, 1) == 0 - } - return atomic.SwapUint32(&ab.value, 0) > 0 -} - -// atomicError is a wrapper for atomically accessed error values -type atomicError struct { - _noCopy noCopy - value atomic.Value -} - -// Set sets the error value regardless of the previous value. -// The value must not be nil -func (ae *atomicError) Set(value error) { - ae.value.Store(value) -} - -// Value returns the current error value -func (ae *atomicError) Value() error { - if v := ae.value.Load(); v != nil { - // this will panic if the value doesn't implement the error interface - return v.(error) - } - return nil -} - -func namedValueToValue(named []driver.NamedValue) ([]driver.Value, error) { - dargs := make([]driver.Value, len(named)) - for n, param := range named { - if len(param.Name) > 0 { - // TODO: support the use of Named Parameters #561 - return nil, errors.New("taosSql: driver does not support the use of Named Parameters") - } - dargs[n] = param.Value - } - return dargs, nil -} - - -/****************************************************************************** -* Utils for C memory issues debugging * -******************************************************************************/ -func SetAllocMode(mode int32, path string) { - cpath := C.CString(path) - defer C.free(unsafe.Pointer(cpath)) - C.taosSetAllocMode(C.int(mode), cpath, false) -} - -func DumpMemoryLeak() { - C.taosDumpMemoryLeak() -} From 9d13fd243bcd9c7fa0ece51b2b7bcb4939697f19 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 11 Apr 2020 04:15:36 +0800 Subject: [PATCH 42/78] Upgrade Python to 3 instead of 2 for testcase development. [TD-139] --- tests/pytest/simpletest.sh | 2 +- tests/pytest/test.py | 1 - tests/pytest/util/log.py | 14 +++++++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/pytest/simpletest.sh b/tests/pytest/simpletest.sh index aab36884f3..0a905beaa1 100755 --- a/tests/pytest/simpletest.sh +++ b/tests/pytest/simpletest.sh @@ -1 +1 @@ -sudo python2 ./test.py -f insert/basic.py +sudo python3 ./test.py -f insert/basic.py diff --git a/tests/pytest/test.py b/tests/pytest/test.py index b88e444665..0a524b44fa 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -73,7 +73,6 @@ if __name__ == "__main__": if fileName == "all": tdCases.runAllLinux(conn) else: - tdLog.info("CBD LN78: %s" % (fileName)) tdCases.runOneLinux(conn, fileName) conn.close() else: diff --git a/tests/pytest/util/log.py b/tests/pytest/util/log.py index 926e582448..e570af85f5 100644 --- a/tests/pytest/util/log.py +++ b/tests/pytest/util/log.py @@ -22,27 +22,27 @@ class TDLog: self.path = "" def info(self, info): - print "%s %s" % (datetime.datetime.now(), info) + print("%s %s" % (datetime.datetime.now(), info)) def sleep(self, sec): - print "%s sleep %d seconds" % (datetime.datetime.now(), sec) + print("%s sleep %d seconds" % (datetime.datetime.now(), sec)) time.sleep(sec) def debug(self, err): - print "\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err) + print("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err)) def success(self, info): - print "\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info) + print("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info)) def notice(self, err): - print "\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err) + print("\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err)) def exit(self, err): - print "\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err) + print("\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err)) sys.exit(1) def printNoPrefix(self, info): - print "\033[1;36m%s\033[0m" % (info) + print("\033[1;36m%s\033[0m" % (info)) tdLog = TDLog() From 1958981b0ed5873b786dac20087ad16b1e463673 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 11 Apr 2020 10:31:48 +0800 Subject: [PATCH 43/78] add const in API --- src/inc/trpc.h | 10 ++++----- src/inc/tsync.h | 44 +++++++++++++++++++++---------------- src/inc/twal.h | 17 +++++++------- src/rpc/src/rpcMain.c | 21 ++++++++++-------- src/vnode/wal/src/walMain.c | 28 +++++++++++------------ 5 files changed, 65 insertions(+), 55 deletions(-) diff --git a/src/inc/trpc.h b/src/inc/trpc.h index a34d107474..5845823b38 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -76,16 +76,16 @@ typedef struct { int (*afp)(char *tableId, char *spi, char *encrypt, char *secret, char *ckey); } SRpcInit; -void *rpcOpen(SRpcInit *pRpc); +void *rpcOpen(const SRpcInit *pRpc); void rpcClose(void *); void *rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); void *rpcReallocCont(void *ptr, int contLen); -void rpcSendRequest(void *thandle, SRpcIpSet *pIpSet, SRpcMsg *pMsg); -void rpcSendResponse(SRpcMsg *pMsg); -void rpcSendRedirectRsp(void *pConn, SRpcIpSet *pIpSet); +void rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg); +void rpcSendResponse(const SRpcMsg *pMsg); +void rpcSendRedirectRsp(void *pConn, const SRpcIpSet *pIpSet); int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); -void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pOut, SRpcMsg *pRsp); +void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, const SRpcMsg *pReq, SRpcMsg *pRsp); #ifdef __cplusplus } diff --git a/src/inc/tsync.h b/src/inc/tsync.h index cdc92b2366..bc28831ecc 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -55,6 +55,24 @@ typedef struct { int role[TAOS_SYNC_MAX_REPLICA]; } SNodesRole; +// if name is null, get the file from index or after, used by master +// if name is provided, get the named file at the specified index, used by unsynced node +// it returns the file magic number and size, if file not there, magic shall be 0. +typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, int32_t *size); + +// get the wal file from index or after +// return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file +typedef int (*FGetWalInfo)(void *ahandle, char *name, uint32_t *index); + +// when a forward pkt is received, call this to handle data +typedef int (*FWriteToCache)(void *ahandle, void *pHead, int type); + +// when forward is confirmed by peer, master call this API to notify app +typedef void (*FConfirmForward)(void *ahandle, void *mhandle, int32_t code); + +// when role is changed, call this to notify app +typedef void (*FNotifyRole)(void *ahandle, int8_t role); + typedef struct { int32_t vgId; // vgroup ID uint64_t version; // initial version @@ -62,31 +80,19 @@ typedef struct { char path[128]; // path to the file void *ahandle; // handle provided by APP + FGetFileInfo getFileInfo; + FGetWalInfo getWalInfo; + FWriteToCache writeToCache; + FConfirmForward confirmForward; + FNotifyRole notifyRole; - // if name is null, get the file from index or after, used by master - // if name is provided, get the named file at the specified index, used by unsynced node - // it returns the file magic number and size, if file not there, magic shall be 0. - uint32_t (*getFileInfo)(void *ahandle, char *name, uint32_t *index, int32_t *size); - - // get the wal file from index or after - // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file - int (*getWalInfo)(void *ahandle, char *name, uint32_t *index); - - // when a forward pkt is received, call this to handle data - int (*writeToCache)(void *ahandle, void *pHead, int type); - - // when forward is confirmed by peer, master call this API to notify app - void (*confirmForward)(void *ahandle, void *mhandle, int32_t code); - - // when role is changed, call this to notify app - void (*notifyRole)(void *ahandle, int8_t role); } SSyncInfo; typedef void* tsync_h; -tsync_h syncStart(SSyncInfo *); +tsync_h syncStart(const SSyncInfo *); void syncStop(tsync_h shandle); -int syncReconfig(tsync_h shandle, SSyncCfg *); +int syncReconfig(tsync_h shandle, const SSyncCfg *); int syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle); void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code); void syncRecover(tsync_h shandle); // recover from other nodes: diff --git a/src/inc/twal.h b/src/inc/twal.h index 53d4f835b0..e90e2e54ce 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -38,15 +38,16 @@ typedef struct { int8_t wals; // number of WAL files; } SWalCfg; -typedef void* twal_h; // WAL HANDLE +typedef void* twalh; // WAL HANDLE +typedef int (*FWalWrite)(void *ahandle, void *pHead, int type); -twal_h walOpen(char *path, SWalCfg *pCfg); -void walClose(twal_h); -int walRenew(twal_h); -int walWrite(twal_h, SWalHead *); -void walFsync(twal_h); -int walRestore(twal_h, void *pVnode, int (*writeFp)(void *ahandle, void *pHead, int type)); -int walGetWalFile(twal_h, char *name, uint32_t *index); +twalh walOpen(const char *path, const SWalCfg *pCfg); +void walClose(twalh); +int walRenew(twalh); +int walWrite(twalh, SWalHead *); +void walFsync(twalh); +int walRestore(twalh, void *pVnode, FWalWrite writeFp); +int walGetWalFile(twalh, char *name, uint32_t *index); extern int wDebugFlag; diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index f1c6deb1d0..bec9621e3b 100755 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -202,7 +202,7 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen); static void rpcLockConn(SRpcConn *pConn); static void rpcUnlockConn(SRpcConn *pConn); -void *rpcOpen(SRpcInit *pInit) { +void *rpcOpen(const SRpcInit *pInit) { SRpcInfo *pRpc; tsRpcMaxRetry = tsRpcMaxTime * 1000 / tsRpcProgressTime; @@ -344,22 +344,22 @@ void *rpcReallocCont(void *ptr, int contLen) { return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); } -void rpcSendRequest(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pMsg) { +void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg) { SRpcInfo *pRpc = (SRpcInfo *)shandle; SRpcReqContext *pContext; - pMsg->contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); + int contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); pContext = (SRpcReqContext *) (pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); pContext->ahandle = pMsg->handle; pContext->pRpc = (SRpcInfo *)shandle; pContext->ipSet = *pIpSet; - pContext->contLen = pMsg->contLen; + pContext->contLen = contLen; pContext->pCont = pMsg->pCont; pContext->msgType = pMsg->msgType; pContext->oldInUse = pIpSet->inUse; pContext->connType = RPC_CONN_UDPC; - if (pMsg->contLen > tsRpcMaxUdpSize) pContext->connType = RPC_CONN_TCPC; + if (contLen > tsRpcMaxUdpSize) pContext->connType = RPC_CONN_TCPC; // connection type is application specific. // for TDengine, all the query, show commands shall have TCP connection @@ -374,11 +374,14 @@ void rpcSendRequest(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pMsg) { return; } -void rpcSendResponse(SRpcMsg *pMsg) { +void rpcSendResponse(const SRpcMsg *pRsp) { int msgLen = 0; - SRpcConn *pConn = (SRpcConn *)pMsg->handle; + SRpcConn *pConn = (SRpcConn *)pRsp->handle; SRpcInfo *pRpc = pConn->pRpc; + SRpcMsg rpcMsg = *pRsp; + SRpcMsg *pMsg = &rpcMsg; + if ( pMsg->pCont == NULL ) { pMsg->pCont = rpcMallocCont(0); pMsg->contLen = 0; @@ -429,7 +432,7 @@ void rpcSendResponse(SRpcMsg *pMsg) { return; } -void rpcSendRedirectRsp(void *thandle, SRpcIpSet *pIpSet) { +void rpcSendRedirectRsp(void *thandle, const SRpcIpSet *pIpSet) { SRpcMsg rpcMsg; rpcMsg.contLen = sizeof(SRpcIpSet); @@ -458,7 +461,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { return 0; } -void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, const SRpcMsg *pMsg, SRpcMsg *pRsp) { SRpcReqContext *pContext; pContext = (SRpcReqContext *) (pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); diff --git a/src/vnode/wal/src/walMain.c b/src/vnode/wal/src/walMain.c index 1738967098..e799bfd63a 100644 --- a/src/vnode/wal/src/walMain.c +++ b/src/vnode/wal/src/walMain.c @@ -48,11 +48,11 @@ typedef struct { int wDebugFlag = 135; static uint32_t walSignature = 0xFAFBFDFE; -static int walHandleExistingFiles(char *path); -static int walRestoreWalFile(char *name, void *pVnode, int (*writeFp)(void *, void *, int)); -static int walRemoveWalFiles(char *path); +static int walHandleExistingFiles(const char *path); +static int walRestoreWalFile(const char *name, void *pVnode, FWalWrite writeFp); +static int walRemoveWalFiles(const char *path); -void *walOpen(char *path, SWalCfg *pCfg) { +void *walOpen(const char *path, const SWalCfg *pCfg) { SWal *pWal = calloc(sizeof(SWal), 1); if (pWal == NULL) return NULL; @@ -82,7 +82,7 @@ void *walOpen(char *path, SWalCfg *pCfg) { void walClose(void *handle) { if (handle == NULL) return; - SWal *pWal = (SWal *)handle; + SWal *pWal = handle; close(pWal->fd); @@ -101,8 +101,8 @@ void walClose(void *handle) { free(pWal); } -int walRenew(twal_h handle) { - SWal *pWal = (SWal *)handle; +int walRenew(void *handle) { + SWal *pWal = handle; int code = 0; pthread_mutex_lock(&pWal->mutex); @@ -144,7 +144,7 @@ int walRenew(twal_h handle) { } int walWrite(void *handle, SWalHead *pHead) { - SWal *pWal = (SWal *)handle; + SWal *pWal = handle; int code = 0; // no wal @@ -164,14 +164,14 @@ int walWrite(void *handle, SWalHead *pHead) { void walFsync(void *handle) { - SWal *pWal = (SWal *)handle; + SWal *pWal = handle; if (pWal->level == TAOS_WAL_FSYNC) fsync(pWal->fd); } int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int)) { - SWal *pWal = (SWal *)handle; + SWal *pWal = handle; int code = 0; struct dirent *ent; int count = 0; @@ -223,7 +223,7 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int)) } int walGetWalFile(void *handle, char *name, uint32_t *index) { - SWal *pWal = (SWal *)handle; + SWal *pWal = handle; int code = 1; int32_t first = 0; @@ -247,7 +247,7 @@ int walGetWalFile(void *handle, char *name, uint32_t *index) { return code; } -static int walRestoreWalFile(char *name, void *pVnode, int (*writeFp)(void *, void *, int)) { +static int walRestoreWalFile(const char *name, void *pVnode, FWalWrite writeFp) { int code = 0; char *buffer = malloc(1024000); // size for one record @@ -293,7 +293,7 @@ static int walRestoreWalFile(char *name, void *pVnode, int (*writeFp)(void *, vo return code; } -int walHandleExistingFiles(char *path) { +int walHandleExistingFiles(const char *path) { int code = 0; char oname[TSDB_FILENAME_LEN * 3]; char nname[TSDB_FILENAME_LEN * 3]; @@ -335,7 +335,7 @@ int walHandleExistingFiles(char *path) { return code; } -static int walRemoveWalFiles(char *path) { +static int walRemoveWalFiles(const char *path) { int plen = strlen(walPrefix); char name[TSDB_FILENAME_LEN * 3]; int code = 0; From da40c92522b71157471254df524ddc4cd204c62a Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 11 Apr 2020 10:45:11 +0800 Subject: [PATCH 44/78] [td-98] fix compiler error --- src/query/src/queryExecutor.c | 2 +- src/util/inc/tutil.h | 8 +------- src/util/src/tglobalcfg.c | 21 +++++++++++---------- src/util/src/tutil.c | 7 +++---- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index e7b1399424..9cb2dcd6c6 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -924,7 +924,7 @@ static void blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStati STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->sid, &win) != TSDB_CODE_SUCCESS) { - return 0; + return; } TSKEY ekey = reviseWindowEkey(pQuery, &win); diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index b7877b4307..c4c802a688 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -121,11 +121,7 @@ int64_t strnatoi(char *num, int32_t len); char* strreplace(const char* str, const char* pattern, const char* rep); -#define POW2(x) ((x) * (x)) - -int32_t strdequote(char *src); - -char *paGetToken(char *src, size_t maxLen, char **token, int32_t *tokenLen); +char *paGetToken(char *src, char **token, int32_t *tokenLen); void taosMsleep(int32_t mseconds); @@ -133,8 +129,6 @@ int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]); int32_t taosHexStrToByteArray(char hexstr[], char bytes[]); -int64_t str2int64(char *str); - int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath); /** diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index a4c034c48d..8ce4530dbf 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -874,12 +874,13 @@ void tsReadGlobalLogConfig() { olen = vlen = 0; getline(&line, &len, fp); - - paGetToken(line, len, &option, &olen); + line[len - 1] = 0; + + paGetToken(line, &option, &olen); if (olen == 0) continue; option[olen] = 0; - paGetToken(option + olen + 1, len, &value, &vlen); + paGetToken(option + olen + 1, &value, &vlen); if (vlen == 0) continue; value[vlen] = 0; @@ -911,18 +912,19 @@ bool tsReadGlobalConfig() { olen = vlen = 0; getline(&line, &len, fp); - - paGetToken(line, len, &option, &olen); + line[len - 1] = 0; + + paGetToken(line, &option, &olen); if (olen == 0) continue; option[olen] = 0; - paGetToken(option + olen + 1, len, &value, &vlen); + paGetToken(option + olen + 1, &value, &vlen); if (vlen == 0) continue; value[vlen] = 0; // For dataDir, the format is: // dataDir /mnt/disk1 0 - paGetToken(value + vlen + 1, len, &value1, &vlen1); + paGetToken(value + vlen + 1, &value1, &vlen1); tsReadConfigOption(option, value); } @@ -1005,11 +1007,10 @@ int tsCfgDynamicOptions(char *msg) { int olen, vlen, code = 0; int vint = 0; - size_t len = 120; - paGetToken(msg, len, &option, &olen); + paGetToken(msg, &option, &olen); if (olen == 0) return TSDB_CODE_INVALID_MSG_CONTENT; - paGetToken(option + olen + 1, len, &value, &vlen); + paGetToken(option + olen + 1, &value, &vlen); if (vlen == 0) vint = 135; else { diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index ba06d73352..5f4c903279 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -164,9 +164,8 @@ char* strtolower(char *dst, const char *src) { return dst; } -char *paGetToken(char *string, size_t maxLen, char **token, int32_t *tokenLen) { +char *paGetToken(char *string, char **token, int32_t *tokenLen) { char quote = 0; - char* start = string; while (*string != 0) { if (*string == ' ' || *string == '\t') { @@ -183,7 +182,7 @@ char *paGetToken(char *string, size_t maxLen, char **token, int32_t *tokenLen) { *token = string; - while (*string != 0 && *string != '\n' && (string - start < maxLen)) { + while (*string != 0) { if (*string == '@' && quote) { //*string = 0; ++string; @@ -337,7 +336,7 @@ int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]) { char hexval[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; for (i = 0; i < len; i++) { - hexstr[i * 2] = hexval[((bytes[i] >> 4) & 0xF)]; + hexstr[i * 2] = hexval[((bytes[i] >> 4u) & 0xF)]; hexstr[(i * 2) + 1] = hexval[(bytes[i]) & 0x0F]; } From 76ea451f15e4994b697ab107e86e0d23766f2f35 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 11 Apr 2020 11:00:33 +0800 Subject: [PATCH 45/78] syncStart definition --- src/vnode/main/src/vnodeMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index e3433d6296..210063ecf5 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -43,7 +43,7 @@ static void vnodeNotifyRole(void *ahandle, int8_t role); static pthread_once_t vnodeModuleInit = PTHREAD_ONCE_INIT; #ifndef _VPEER -tsync_h syncStart(SSyncInfo *info) { return NULL; } +tsync_h syncStart(const SSyncInfo *info) { return NULL; } int syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle) { return 0; } #endif From 0039bd873ab95ea282ec4097d331426f87521b66 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 11 Apr 2020 11:12:48 +0800 Subject: [PATCH 46/78] fix duplicate commit key --- src/vnode/tsdb/src/tsdbMain.c | 4 +- src/vnode/tsdb/tests/tsdbTests.cpp | 150 +++++++++++++++-------------- 2 files changed, 79 insertions(+), 75 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 62a43333ae..6873c69c2a 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -763,6 +763,8 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max int numOfRows = 0; do { + if (numOfRows >= maxRowsToRead) break; + SSkipListNode *node = tSkipListIterGet(pIter); if (node == NULL) break; @@ -770,9 +772,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max if (dataRowKey(row) > maxKey) break; tdAppendDataRowToDataCol(row, pCols); - numOfRows++; - if (numOfRows >= maxRowsToRead) break; } while (tSkipListIterNext(pIter)); return numOfRows; diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp index 9ee49d6a70..eac29af097 100644 --- a/src/vnode/tsdb/tests/tsdbTests.cpp +++ b/src/vnode/tsdb/tests/tsdbTests.cpp @@ -48,104 +48,104 @@ TEST(TsdbTest, DISABLED_tableEncodeDecode) { ASSERT_EQ(memcmp(pTable->schema, tTable->schema, sizeof(STSchema) + sizeof(STColumn) * nCols), 0); } -// TEST(TsdbTest, DISABLED_createRepo) { -TEST(TsdbTest, createRepo) { - STsdbCfg config; +TEST(TsdbTest, DISABLED_createRepo) { +// TEST(TsdbTest, createRepo) { + // STsdbCfg config; - // 1. Create a tsdb repository - tsdbSetDefaultCfg(&config); - tsdb_repo_t *pRepo = tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL); - ASSERT_NE(pRepo, nullptr); + // // 1. Create a tsdb repository + // tsdbSetDefaultCfg(&config); + // tsdb_repo_t *pRepo = tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL); + // ASSERT_NE(pRepo, nullptr); - // 2. Create a normal table - STableCfg tCfg; - ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_SUPER_TABLE, 987607499877672L, 0), -1); - ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_NORMAL_TABLE, 987607499877672L, 0), 0); + // // 2. Create a normal table + // STableCfg tCfg; + // ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_SUPER_TABLE, 987607499877672L, 0), -1); + // ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_NORMAL_TABLE, 987607499877672L, 0), 0); - int nCols = 5; - STSchema *schema = tdNewSchema(nCols); + // int nCols = 5; + // STSchema *schema = tdNewSchema(nCols); - for (int i = 0; i < nCols; i++) { - if (i == 0) { - tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1); - } else { - tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1); - } - } + // for (int i = 0; i < nCols; i++) { + // if (i == 0) { + // tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1); + // } else { + // tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1); + // } + // } - tsdbTableSetSchema(&tCfg, schema, true); + // tsdbTableSetSchema(&tCfg, schema, true); - tsdbCreateTable(pRepo, &tCfg); + // tsdbCreateTable(pRepo, &tCfg); - // // 3. Loop to write some simple data - int nRows = 1; - int rowsPerSubmit = 1; - int64_t start_time = 1584081000000; + // // // 3. Loop to write some simple data + // int nRows = 1; + // int rowsPerSubmit = 1; + // int64_t start_time = 1584081000000; - SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * rowsPerSubmit); + // SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * rowsPerSubmit); - double stime = getCurTime(); + // double stime = getCurTime(); - for (int k = 0; k < nRows/rowsPerSubmit; k++) { - memset((void *)pMsg, 0, sizeof(SSubmitMsg)); - SSubmitBlk *pBlock = pMsg->blocks; - pBlock->uid = 987607499877672L; - pBlock->tid = 0; - pBlock->sversion = 0; - pBlock->len = 0; - for (int i = 0; i < rowsPerSubmit; i++) { - // start_time += 1000; - start_time += 1000; - SDataRow row = (SDataRow)(pBlock->data + pBlock->len); - tdInitDataRow(row, schema); + // for (int k = 0; k < nRows/rowsPerSubmit; k++) { + // memset((void *)pMsg, 0, sizeof(SSubmitMsg)); + // SSubmitBlk *pBlock = pMsg->blocks; + // pBlock->uid = 987607499877672L; + // pBlock->tid = 0; + // pBlock->sversion = 0; + // pBlock->len = 0; + // for (int i = 0; i < rowsPerSubmit; i++) { + // // start_time += 1000; + // start_time += 1000; + // SDataRow row = (SDataRow)(pBlock->data + pBlock->len); + // tdInitDataRow(row, schema); - for (int j = 0; j < schemaNCols(schema); j++) { - if (j == 0) { // Just for timestamp - tdAppendColVal(row, (void *)(&start_time), schemaColAt(schema, j)); - } else { // For int - int val = 10; - tdAppendColVal(row, (void *)(&val), schemaColAt(schema, j)); - } - } - pBlock->len += dataRowLen(row); - } - pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len; - pMsg->numOfBlocks = 1; + // for (int j = 0; j < schemaNCols(schema); j++) { + // if (j == 0) { // Just for timestamp + // tdAppendColVal(row, (void *)(&start_time), schemaColAt(schema, j)); + // } else { // For int + // int val = 10; + // tdAppendColVal(row, (void *)(&val), schemaColAt(schema, j)); + // } + // } + // pBlock->len += dataRowLen(row); + // } + // pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len; + // pMsg->numOfBlocks = 1; - pBlock->len = htonl(pBlock->len); - pBlock->numOfRows = htonl(pBlock->numOfRows); - pBlock->uid = htobe64(pBlock->uid); - pBlock->tid = htonl(pBlock->tid); + // pBlock->len = htonl(pBlock->len); + // pBlock->numOfRows = htonl(pBlock->numOfRows); + // pBlock->uid = htobe64(pBlock->uid); + // pBlock->tid = htonl(pBlock->tid); - pBlock->sversion = htonl(pBlock->sversion); - pBlock->padding = htonl(pBlock->padding); + // pBlock->sversion = htonl(pBlock->sversion); + // pBlock->padding = htonl(pBlock->padding); - pMsg->length = htonl(pMsg->length); - pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); - pMsg->compressed = htonl(pMsg->numOfBlocks); + // pMsg->length = htonl(pMsg->length); + // pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + // pMsg->compressed = htonl(pMsg->numOfBlocks); - tsdbInsertData(pRepo, pMsg); - } + // tsdbInsertData(pRepo, pMsg); + // } - double etime = getCurTime(); + // double etime = getCurTime(); - void *ptr = malloc(150000); - free(ptr); + // void *ptr = malloc(150000); + // free(ptr); - printf("Spent %f seconds to write %d records\n", etime - stime, nRows); + // printf("Spent %f seconds to write %d records\n", etime - stime, nRows); - tsdbCloseRepo(pRepo); + // tsdbCloseRepo(pRepo); } // TEST(TsdbTest, DISABLED_openRepo) { TEST(TsdbTest, openRepo) { - tsdb_repo_t *repo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL); + tsdb_repo_t *repo = tsdbOpenRepo("/home/ubuntu/work/build/test/data/vnode/vnode2/tsdb", NULL); ASSERT_NE(repo, nullptr); STsdbRepo *pRepo = (STsdbRepo *)repo; - SFileGroup *pGroup = tsdbSearchFGroup(pRepo->tsdbFileH, 1833); + SFileGroup *pGroup = tsdbSearchFGroup(pRepo->tsdbFileH, 1655); for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { tsdbOpenFile(&pGroup->files[type], O_RDONLY); @@ -156,7 +156,7 @@ TEST(TsdbTest, openRepo) { SCompInfo *pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + pIdx[1].len); - tsdbLoadCompBlocks(pGroup, &pIdx[0], (void *)pCompInfo); + tsdbLoadCompBlocks(pGroup, &pIdx[1], (void *)pCompInfo); int blockIdx = 0; SCompBlock *pBlock = &(pCompInfo->blocks[blockIdx]); @@ -166,11 +166,15 @@ TEST(TsdbTest, openRepo) { tsdbLoadCompCols(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, (void *)pCompData); STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, pCompData->uid); - SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(pTable->schema), 5, 10); - tdInitDataCols(pDataCols, pTable->schema); + SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(tsdbGetTableSchema(pRepo->tsdbMeta, pTable)), 5, 10); + tdInitDataCols(pDataCols, tsdbGetTableSchema(pRepo->tsdbMeta, pTable)); tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, 1, pDataCols, pCompData); + tdResetDataCols(pDataCols); + + tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock + 1, 1, pDataCols, pCompData); + int k = 0; From 5fd9705d45b03ac1523f88cea1636f9ed746a7ad Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 11 Apr 2020 11:30:26 +0800 Subject: [PATCH 47/78] [td-98] fix bugs while query/insert mixed up in sql requests. --- src/client/inc/tsclient.h | 26 +++++++------- src/client/src/tscAsync.c | 6 ++-- src/client/src/tscParseInsert.c | 60 +++++++++++++++------------------ src/client/src/tscServer.c | 4 +-- src/client/src/tscSql.c | 30 +++++++++-------- src/client/src/tscUtil.c | 18 +++++----- 6 files changed, 71 insertions(+), 73 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 5c442a155b..0d684b72dc 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -267,19 +267,19 @@ typedef struct { int32_t numOfTablesInSubmit; }; - int32_t clauseIndex; // index of multiple subclause query - int8_t isParseFinish; - short numOfCols; - uint32_t allocSize; - char * payload; - int payloadLen; - + int32_t clauseIndex; // index of multiple subclause query + int8_t parseFinished; + short numOfCols; + uint32_t allocSize; + char * payload; + int32_t payloadLen; SQueryInfo **pQueryInfo; int32_t numOfClause; - - // submit data blocks branched according to vnode - SDataBlockList *pDataBlocks; - + + SDataBlockList *pDataBlocks; // submit data blocks after parsing sql + char * curSql; // current sql, resume position of sql after parsing paused + void * pTableList; // referred table involved in sql + // for parameter ('?') binding and batch processing int32_t batchSize; int32_t numOfParams; @@ -358,8 +358,6 @@ typedef struct SSqlObj { SSqlCmd cmd; SSqlRes res; uint8_t numOfSubs; - char * asyncTblPos; - void * pTableHashList; struct SSqlObj **pSubs; struct SSqlObj * prev, *next; } SSqlObj; @@ -422,7 +420,7 @@ void tscRestoreSQLFunctionForMetricQuery(SQueryInfo *pQueryInfo); int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); void tscDestroyResPointerInfo(SSqlRes *pRes); -void tscFreeSqlCmdData(SSqlCmd *pCmd); +void tscResetSqlCmdObj(SSqlCmd *pCmd); void tscFreeResData(SSqlObj *pSql); /** diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index e672225308..25f5856877 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -67,7 +67,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const pRes->numOfRows = 1; strtolower(pSql->sqlstr, sqlstr); - tscDump("%p pObj:%p, Async SQL: %s", pSql, pObj, pSql->sqlstr); + tscDump("%p SQL: %s", pSql, pSql->sqlstr); int32_t code = tsParseSql(pSql, true); if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; @@ -342,7 +342,7 @@ void tscProcessAsyncRes(SSchedMsg *pMsg) { (*pSql->fp)(pSql->param, taosres, code); if (shouldFree) { - tscTrace("%p Async sql is automatically freed in async res", pSql); + tscTrace("%p sqlObj is automatically freed in async res", pSql); tscFreeSqlObj(pSql); } } @@ -463,7 +463,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; } else { // normal async query continues - if (pCmd->isParseFinish) { + if (pCmd->parseFinished) { tscTrace("%p re-send data to vnode in table Meta callback since sql parsed completed", pSql); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 518b3680b4..c4d3c475c4 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -21,7 +21,6 @@ #include "os.h" #include "hash.h" -//#include "tscSecondaryMerge.h" #include "tscUtil.h" #include "tschemautil.h" #include "tsclient.h" @@ -656,7 +655,7 @@ void sortRemoveDuplicates(STableDataBlocks *dataBuf) { } } -static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char **str, SParsedDataColInfo *spd, +static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **str, SParsedDataColInfo *spd, int32_t *totalNum) { SSqlCmd * pCmd = &pSql->cmd; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); @@ -664,7 +663,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char STableComInfo tinfo = tscGetTableInfo(pTableMeta); STableDataBlocks *dataBuf = NULL; - int32_t ret = tscGetDataBlockFromList(pTableHashList, pCmd->pDataBlocks, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, + int32_t ret = tscGetDataBlockFromList(pTableList, pCmd->pDataBlocks, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), tinfo.rowSize, pTableMetaInfo->name, pTableMeta, &dataBuf); if (ret != TSDB_CODE_SUCCESS) { @@ -942,7 +941,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } code = tscGetTableMeta(pSql, pTableMetaInfo); - if (pSql->asyncTblPos == NULL) { + if (pCmd->curSql == NULL) { assert(code == TSDB_CODE_ACTION_IN_PROGRESS); } } @@ -1008,23 +1007,23 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { return code; } - assert(((NULL == pSql->asyncTblPos) && (NULL == pSql->pTableHashList)) - || ((NULL != pSql->asyncTblPos) && (NULL != pSql->pTableHashList))); + assert(((NULL == pCmd->curSql) && (NULL == pCmd->pTableList)) + || ((NULL != pCmd->curSql) && (NULL != pCmd->pTableList))); - if ((NULL == pSql->asyncTblPos) && (NULL == pSql->pTableHashList)) { - pSql->pTableHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); + if ((NULL == pCmd->curSql) && (NULL == pCmd->pTableList)) { + pCmd->pTableList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); pSql->cmd.pDataBlocks = tscCreateBlockArrayList(); - if (NULL == pSql->pTableHashList || NULL == pSql->cmd.pDataBlocks) { + if (NULL == pCmd->pTableList || NULL == pSql->cmd.pDataBlocks) { code = TSDB_CODE_CLI_OUT_OF_MEMORY; goto _error_clean; } } else { - assert((NULL != pSql->asyncTblPos) && (NULL != pSql->pTableHashList)); - str = pSql->asyncTblPos; + assert((NULL != pCmd->curSql) && (NULL != pCmd->pTableList)); + str = pCmd->curSql; } - tscTrace("%p create data block list for submit data:%p, asyncTblPos:%p, pTableHashList:%p", pSql, pSql->cmd.pDataBlocks, pSql->asyncTblPos, pSql->pTableHashList); + tscTrace("%p create data block list for submit data:%p, curSql:%p, pTableList:%p", pSql, pSql->cmd.pDataBlocks, pCmd->curSql, pCmd->pTableList); while (1) { int32_t index = 0; @@ -1052,7 +1051,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { } } - pSql->asyncTblPos = sToken.z; + pCmd->curSql = sToken.z; // Check if the table name available or not if (validateTableName(sToken.z, sToken.n) != TSDB_CODE_SUCCESS) { @@ -1064,7 +1063,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { goto _error_clean; } - ptrdiff_t pos = pSql->asyncTblPos - pSql->sqlstr; + ptrdiff_t pos = pCmd->curSql - pSql->sqlstr; if ((code = tscCheckIfCreateTable(&str, pSql)) != TSDB_CODE_SUCCESS) { /* @@ -1075,13 +1074,13 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { */ if (TSDB_CODE_ACTION_IN_PROGRESS == code) { tscTrace("%p waiting for get table meta during insert, then resume from offset: %" PRId64 " , %s", pSql, - pos, pSql->asyncTblPos); + pos, pCmd->curSql); return code; } // todo add to return tscError("%p async insert parse error, code:%d, %s", pSql, code, tstrerror(code)); - pSql->asyncTblPos = NULL; + pCmd->curSql = NULL; goto _error_clean; // TODO: should _clean or _error_clean to async flow ???? } @@ -1115,7 +1114,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { * app here insert data in different vnodes, so we need to set the following * data in another submit procedure using async insert routines */ - code = doParseInsertStatement(pSql, pSql->pTableHashList, &str, &spd, &totalNum); + code = doParseInsertStatement(pSql, pCmd->pTableList, &str, &spd, &totalNum); if (code != TSDB_CODE_SUCCESS) { goto _error_clean; } @@ -1227,7 +1226,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { goto _error_clean; } - code = doParseInsertStatement(pSql, pSql->pTableHashList, &str, &spd, &totalNum); + code = doParseInsertStatement(pSql, pCmd->pTableList, &str, &spd, &totalNum); if (code != TSDB_CODE_SUCCESS) { goto _error_clean; } @@ -1257,11 +1256,11 @@ _error_clean: pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); _clean: - taosHashCleanup(pSql->pTableHashList); + taosHashCleanup(pCmd->pTableList); + pCmd->pTableList = NULL; - pSql->pTableHashList = NULL; - pSql->asyncTblPos = NULL; - pCmd->isParseFinish = 1; + pCmd->curSql = NULL; + pCmd->parseFinished = 1; return code; } @@ -1305,17 +1304,15 @@ int tsParseSql(SSqlObj *pSql, bool initialParse) { tscFreeSqlObjPartial(pSql); pSql->sqlstr = p; } else { - tscTrace("continue parse sql: %s", pSql->asyncTblPos); + tscTrace("continue parse sql: %s", pSql->cmd.curSql); } if (tscIsInsertOrImportData(pSql->sqlstr)) { /* - * only for async multi-vnode insertion - * Set the fp before parse the sql string, in case of getmetermeta failed, in which - * the error handle callback function can rightfully restore the user defined function (fp) + * Set the fp before parse the sql string, in case of getTableMeta failed, in which + * the error handle callback function can rightfully restore the user-defined callback function (fp). */ if (initialParse) { - // replace user defined callback function with multi-insert proxy function pSql->fetchFp = pSql->fp; pSql->fp = (void(*)())tscHandleMultivnodeInsert; } @@ -1335,11 +1332,11 @@ int tsParseSql(SSqlObj *pSql, bool initialParse) { } /* - * the pRes->code may be modified or even released by another thread in tscTableMetaCallBack - * function, so do NOT use pRes->code to determine if the getMeterMeta/getMetricMeta function - * invokes new threads to get data from mnode or simply retrieves data from cache. + * the pRes->code may be modified or released by another thread in tscTableMetaCallBack function, + * so do NOT use pRes->code to determine if the getTableMeta/getMetricMeta function + * invokes new threads to get data from mgmt node or simply retrieves data from cache. * - * do NOT assign return code to pRes->code for the same reason for it may be released by another thread + * do NOT assign return code to pRes->code for the same reason since it may be released by another thread * pRes->code = ret; */ return ret; @@ -1457,7 +1454,6 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) { return numOfRows; } -// multi-vnodes insertion in sync query model void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; if (pCmd->command != TSDB_SQL_INSERT) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 0bb3cb0dc7..ebac676b77 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -350,7 +350,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { void *taosres = tscKeepConn[pCmd->command] ? pSql : NULL; rpcMsg->code = pRes->code ? pRes->code : pRes->numOfRows; - tscTrace("%p Async SQL result:%s res:%p", pSql, tstrerror(pRes->code), pSql); + tscTrace("%p SQL result:%s res:%p", pSql, tstrerror(pRes->code), pSql); /* * Whether to free sqlObj or not should be decided before call the user defined function, since this SqlObj @@ -364,7 +364,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { (*pSql->fp)(pSql->param, taosres, rpcMsg->code); if (shouldFree) { - tscTrace("%p Async sql is automatically freed", pSql); + tscTrace("%p sqlObj is automatically freed", pSql); tscFreeSqlObj(pSql); } } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index c522dbfebe..1eee5d5e7f 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -232,15 +232,16 @@ void taos_close(TAOS *taos) { int taos_query_imp(STscObj *pObj, SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; - - pRes->numOfRows = 1; + SSqlCmd *pCmd = &pSql->cmd; + + pRes->numOfRows = 1; pRes->numOfTotal = 0; pRes->numOfTotalInCurrentClause = 0; - pSql->asyncTblPos = NULL; - if (NULL != pSql->pTableHashList) { - taosHashCleanup(pSql->pTableHashList); - pSql->pTableHashList = NULL; + pCmd->curSql = NULL; + if (NULL != pCmd->pTableList) { + taosHashCleanup(pCmd->pTableList); + pCmd->pTableList = NULL; } tscDump("%p pObj:%p, SQL: %s", pSql, pObj, pSql->sqlstr); @@ -767,7 +768,7 @@ void taos_free_result_imp(TAOS_RES *res, int keepCmd) { tscTrace("%p qhandle is null, abort free, fp:%p", pSql, pSql->fp); if (tscShouldFreeAsyncSqlObj(pSql)) { - tscTrace("%p Async SqlObj is freed by app", pSql); + tscTrace("%p SqlObj is freed by app", pSql); tscFreeSqlObj(pSql); } else { if (keepCmd) { @@ -851,7 +852,7 @@ void taos_free_result_imp(TAOS_RES *res, int keepCmd) { assert(pRes->numOfRows == 0 || (pCmd->command > TSDB_SQL_LOCAL)); tscFreeSqlObj(pSql); - tscTrace("%p Async sql result is freed by app", pSql); + tscTrace("%p sql result is freed by app", pSql); } else { if (keepCmd) { tscFreeSqlResult(pSql); @@ -1027,8 +1028,9 @@ int taos_validate_sql(TAOS *taos, const char *sql) { SSqlObj *pSql = pObj->pSql; SSqlRes *pRes = &pSql->res; - - pRes->numOfRows = 1; + SSqlCmd *pCmd = &pSql->cmd; + + pRes->numOfRows = 1; pRes->numOfTotal = 0; pRes->numOfTotalInCurrentClause = 0; @@ -1051,10 +1053,10 @@ int taos_validate_sql(TAOS *taos, const char *sql) { strtolower(pSql->sqlstr, sql); - pSql->asyncTblPos = NULL; - if (NULL != pSql->pTableHashList) { - taosHashCleanup(pSql->pTableHashList); - pSql->pTableHashList = NULL; + pCmd->curSql = NULL; + if (NULL != pCmd->pTableList) { + taosHashCleanup(pCmd->pTableList); + pCmd->pTableList = NULL; } pRes->code = (uint8_t)tsParseSql(pSql, false); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 2385b4ad82..7ad7e65c51 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -406,10 +406,16 @@ void tscDestroyResPointerInfo(SSqlRes* pRes) { pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free } -void tscFreeSqlCmdData(SSqlCmd* pCmd) { - pCmd->command = 0; +void tscResetSqlCmdObj(SSqlCmd* pCmd) { + pCmd->command = 0; pCmd->numOfCols = 0; - pCmd->count = 0; + pCmd->count = 0; + pCmd->curSql = NULL; + pCmd->msgType = 0; + pCmd->parseFinished = 0; + + taosHashCleanup(pCmd->pTableList); + pCmd->pTableList= NULL; pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); tscFreeSubqueryInfo(pCmd); @@ -480,14 +486,10 @@ void tscFreeSqlObjPartial(SSqlObj* pSql) { tscFreeSqlResult(pSql); tfree(pSql->pSubs); - taosHashCleanup(pSql->pTableHashList); - pSql->freed = 0; pSql->numOfSubs = 0; - pSql->pTableHashList = NULL; - pSql->asyncTblPos = NULL; - tscFreeSqlCmdData(pCmd); + tscResetSqlCmdObj(pCmd); tscTrace("%p partially free sqlObj completed", pSql); } From 3e9e46491bbb2ce98a528e3f11a08489d94ce0e9 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 11 Apr 2020 11:34:05 +0800 Subject: [PATCH 48/78] use private lock instead of mutex --- src/util/src/ihash.c | 81 +++++++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 28 deletions(-) diff --git a/src/util/src/ihash.c b/src/util/src/ihash.c index e1c7cb0e58..f8697086e4 100644 --- a/src/util/src/ihash.c +++ b/src/util/src/ihash.c @@ -16,7 +16,7 @@ #include "os.h" typedef struct _str_node_t { - uint64_t key; + uint64_t key; struct _str_node_t *prev; struct _str_node_t *next; char data[]; @@ -24,6 +24,7 @@ typedef struct _str_node_t { typedef struct { IHashNode **hashList; + int64_t *lockedBy; int32_t maxSessions; int32_t dataSize; int32_t (*hashFp)(void *, uint64_t key); @@ -36,6 +37,9 @@ int32_t taosHashInt(void *handle, uint64_t key) { return hash; } +static void taosLockIntHash(IHashObj *pObj, int hash); +static void taosUnlockIntHash(IHashObj *pObj, int hash); + char *taosAddIntHash(void *handle, uint64_t key, char *pData) { int32_t hash; IHashNode *pNode; @@ -50,7 +54,7 @@ char *taosAddIntHash(void *handle, uint64_t key, char *pData) { if (pNode == NULL) return NULL; - pthread_mutex_lock(&pObj->mutex); + taosLockIntHash(pObj, hash); pNode->key = key; if (pData != NULL) { @@ -62,7 +66,7 @@ char *taosAddIntHash(void *handle, uint64_t key, char *pData) { if (pObj->hashList[hash] != 0) (pObj->hashList[hash])->prev = pNode; pObj->hashList[hash] = pNode; - pthread_mutex_unlock(&pObj->mutex); + taosUnlockIntHash(pObj, hash); return (char *)pNode->data; } @@ -77,7 +81,7 @@ void taosDeleteIntHash(void *handle, uint64_t key) { hash = (*(pObj->hashFp))(pObj, key); - pthread_mutex_lock(&pObj->mutex); + taosLockIntHash(pObj, hash); pNode = pObj->hashList[hash]; while (pNode) { @@ -100,7 +104,7 @@ void taosDeleteIntHash(void *handle, uint64_t key) { free(pNode); } - pthread_mutex_unlock(&pObj->mutex); + taosUnlockIntHash(pObj, hash); } char *taosGetIntHashData(void *handle, uint64_t key) { @@ -113,7 +117,7 @@ char *taosGetIntHashData(void *handle, uint64_t key) { hash = (*pObj->hashFp)(pObj, key); - pthread_mutex_lock(&pObj->mutex); + taosLockIntHash(pObj, hash); pNode = pObj->hashList[hash]; @@ -125,7 +129,7 @@ char *taosGetIntHashData(void *handle, uint64_t key) { pNode = pNode->next; } - pthread_mutex_unlock(&pObj->mutex); + taosUnlockIntHash(pObj, hash); if (pNode) return pNode->data; @@ -152,7 +156,12 @@ void *taosInitIntHash(int32_t maxSessions, int32_t dataSize, int32_t (*fp)(void } memset(pObj->hashList, 0, sizeof(IHashNode *) * (size_t)maxSessions); - pthread_mutex_init(&pObj->mutex, NULL); + pObj->lockedBy = (int64_t *)calloc(sizeof(int64_t), maxSessions); + if (pObj->lockedBy == NULL) { + free(pObj); + free(pObj->hashList); + pObj = NULL; + } return pObj; } @@ -164,26 +173,25 @@ void taosCleanUpIntHash(void *handle) { pObj = (IHashObj *)handle; if (pObj == NULL || pObj->maxSessions <= 0) return; - pthread_mutex_lock(&pObj->mutex); - if (pObj->hashList) { for (int32_t i = 0; i < pObj->maxSessions; ++i) { + taosLockIntHash(pObj, i); + pNode = pObj->hashList[i]; while (pNode) { pNext = pNode->next; free(pNode); pNode = pNext; } + + taosUnlockIntHash(pObj, i); } free(pObj->hashList); } - pthread_mutex_unlock(&pObj->mutex); - - pthread_mutex_destroy(&pObj->mutex); - memset(pObj, 0, sizeof(IHashObj)); + free(pObj->lockedBy); free(pObj); } @@ -194,10 +202,10 @@ void taosCleanUpIntHashWithFp(void *handle, void (*fp)(char *)) { pObj = (IHashObj *)handle; if (pObj == NULL || pObj->maxSessions <= 0) return; - pthread_mutex_lock(&pObj->mutex); - if (pObj->hashList) { for (int i = 0; i < pObj->maxSessions; ++i) { + taosLockIntHash(pObj, i); + pNode = pObj->hashList[i]; while (pNode) { pNext = pNode->next; @@ -205,15 +213,13 @@ void taosCleanUpIntHashWithFp(void *handle, void (*fp)(char *)) { free(pNode); pNode = pNext; } + + taosUnlockIntHash(pObj, i); } free(pObj->hashList); } - pthread_mutex_unlock(&pObj->mutex); - - pthread_mutex_destroy(&pObj->mutex); - memset(pObj, 0, sizeof(IHashObj)); free(pObj); } @@ -225,20 +231,20 @@ void taosVisitIntHashWithFp(void *handle, int (*fp)(char *, void *), void *param pObj = (IHashObj *)handle; if (pObj == NULL || pObj->maxSessions <= 0) return; - pthread_mutex_lock(&pObj->mutex); - if (pObj->hashList) { for (int i = 0; i < pObj->maxSessions; ++i) { + taosLockIntHash(pObj, i); + pNode = pObj->hashList[i]; while (pNode) { pNext = pNode->next; (*fp)(pNode->data, param); pNode = pNext; } + + taosUnlockIntHash(pObj, i); } } - - pthread_mutex_unlock(&pObj->mutex); } int32_t taosGetIntHashSize(void *handle) { @@ -249,19 +255,38 @@ int32_t taosGetIntHashSize(void *handle) { pObj = (IHashObj *)handle; if (pObj == NULL || pObj->maxSessions <= 0) return 0; - pthread_mutex_lock(&pObj->mutex); - if (pObj->hashList) { for (int i = 0; i < pObj->maxSessions; ++i) { + taosLockIntHash(pObj, i); + pNode = pObj->hashList[i]; while (pNode) { pNext = pNode->next; num++; pNode = pNext; } + + taosUnlockIntHash(pObj, i); } } - pthread_mutex_unlock(&pObj->mutex); return num; -} \ No newline at end of file +} + +static void taosLockIntHash(IHashObj *pObj, int hash) { + int64_t tid = taosGetPthreadId(); + int i = 0; + while (atomic_val_compare_exchange_64(&(pObj->lockedBy[hash]), 0, tid) != 0) { + if (++i % 1000 == 0) { + sched_yield(); + } + } +} + +static void taosUnlockIntHash(IHashObj *pObj, int hash) { + int64_t tid = taosGetPthreadId(); + if (atomic_val_compare_exchange_64(&(pObj->lockedBy[hash]), tid, 0) != tid) { + assert(false); + } +} + From 1a5b1c56141b157120ef9d6f592a103e5627563a Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 11 Apr 2020 11:42:40 +0800 Subject: [PATCH 49/78] remove mutex --- src/util/src/ihash.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/util/src/ihash.c b/src/util/src/ihash.c index f8697086e4..30773ae8d9 100644 --- a/src/util/src/ihash.c +++ b/src/util/src/ihash.c @@ -28,7 +28,6 @@ typedef struct { int32_t maxSessions; int32_t dataSize; int32_t (*hashFp)(void *, uint64_t key); - pthread_mutex_t mutex; } IHashObj; int32_t taosHashInt(void *handle, uint64_t key) { From 90e6fb395e86306d9589a2d310c955656086d996 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 11 Apr 2020 12:50:28 +0800 Subject: [PATCH 50/78] [td-98] fix bugs in interval query --- src/query/src/queryExecutor.c | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 9cb2dcd6c6..1ba74af2be 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -2566,7 +2566,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { tsdb_query_handle_t pQueryHandle = pRuntimeEnv->pQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { - // check if query is killed or not set the status of query to pass the status check + if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { return 0; } @@ -3865,7 +3865,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde int32_t startIdx = 0; int32_t step = -1; - dTrace("QInfo:%p start to copy data from windowResInfo to pQuery buf", GET_QINFO_ADDR(pQuery)); + dTrace("QInfo:%p start to copy data from windowResInfo to query buf", GET_QINFO_ADDR(pQuery)); int32_t totalSubset = getNumOfSubset(pQInfo); if (orderType == TSDB_ORDER_ASC) { @@ -3914,7 +3914,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde } } - dTrace("QInfo:%p copy data to SQuery buf completed", pQInfo); + dTrace("QInfo:%p copy data to query buf completed", pQInfo); #ifdef _DEBUG_VIEW displayInterResult(pQuery->sdata, pQuery, numOfResult); @@ -4830,7 +4830,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { // vnodePrintQueryStatistics(pSupporter); } - dTrace("QInfo:%p current:%lldd, total:%lldd", pQInfo, pQuery->rec.rows, pQuery->rec.total); + dTrace("QInfo:%p current:%lld, total:%lld", pQInfo, pQuery->rec.rows, pQuery->rec.total); return; } @@ -4909,8 +4909,6 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { // since the numOfOutputElems must be identical for all sql functions that are allowed to be executed simutanelously. pQuery->rec.rows = getNumOfResult(pRuntimeEnv); - // assert(pQuery->size <= pQuery->pointsToRead && - // Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED)); // must be top/bottom query if offset > 0 if (pQuery->limit.offset > 0) { @@ -4977,7 +4975,7 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; while (1) { - initCtxOutputBuf(pRuntimeEnv); +// initCtxOutputBuf(pRuntimeEnv); scanAllDataBlocks(pRuntimeEnv); if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { @@ -5058,13 +5056,7 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); } - pQuery->rec.rows += pQuery->rec.rows; pQInfo->pointsInterpo += numOfInterpo; - - // dTrace("%p vid:%d sid:%d id:%s, %d points returned %d points interpo, totalRead:%d totalInterpo:%d - // totalReturn:%d", - // pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->size, numOfInterpo, - // pQInfo->size - pQInfo->pointsInterpo, pQInfo->pointsInterpo, pQInfo->pointsReturned); } static void tableQueryImpl(SQInfo* pQInfo) { @@ -5129,7 +5121,6 @@ static void tableQueryImpl(SQInfo* pQInfo) { if (isIntervalQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) tableIntervalProcessor(pQInfo); } else if (isFixedOutputQuery(pQuery)) { - assert(pQuery->checkBuffer == 0); tableFixedOutputProcessor(pQInfo); } else { // diff/add/multiply/subtract/division assert(pQuery->checkBuffer == 1); @@ -5139,13 +5130,15 @@ static void tableQueryImpl(SQInfo* pQInfo) { // record the total elapsed time pQInfo->elapsedTime += (taosGetTimestampUs() - st); + assert(taosArrayGetSize(pQInfo->pTableIdList) == 1); /* check if query is killed or not */ if (isQueryKilled(pQInfo)) { dTrace("QInfo:%p query is killed", pQInfo); } else { - dTrace("QInfo:%p query task completed, %" PRId64 " rows will returned, total:%" PRId64 " rows", pQInfo, pQuery->rec.rows, - pQuery->rec.total); + STableId* pTableId = taosArrayGet(pQInfo->pTableIdList, 0); + dTrace("QInfo:%p uid:%" PRIu64 " tid:%d, query completed, %" PRId64 " rows returned, numOfTotal:%" PRId64 " rows", + pQInfo, pTableId->uid, pTableId->tid, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } sem_post(&pQInfo->dataReady); From 228cb6f974b0f02d8b5fb5817df52c6b103c3a17 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 11 Apr 2020 14:47:37 +0800 Subject: [PATCH 51/78] [td-98] fix compData buffer overflow --- src/common/inc/name.h | 4 +-- src/query/inc/queryExecutor.h | 4 +-- src/query/src/queryExecutor.c | 54 ++++++++++++++++------------------- src/vnode/tsdb/inc/tsdb.h | 2 +- src/vnode/tsdb/src/tsdbRead.c | 38 +++++++++++++++--------- 5 files changed, 54 insertions(+), 48 deletions(-) diff --git a/src/common/inc/name.h b/src/common/inc/name.h index bafa003609..a629128324 100644 --- a/src/common/inc/name.h +++ b/src/common/inc/name.h @@ -14,10 +14,10 @@ typedef struct SDataStatis { int16_t numOfNull; } SDataStatis; -typedef struct SColumnInfoEx { +typedef struct SColumnInfoData { SColumnInfo info; void* pData; // the corresponding block data in memory -} SColumnInfoEx; +} SColumnInfoData; void extractTableName(const char *tableId, char *name); diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index dc5f2fcb78..5aab3abf40 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -89,7 +89,7 @@ typedef struct SColumnFilterElem { } SColumnFilterElem; typedef struct SSingleColumnFilterInfo { - SColumnInfoEx info; + SColumnInfoData info; int32_t numOfFilters; SColumnFilterElem* pFilters; void* pData; @@ -130,7 +130,7 @@ typedef struct SQuery { int32_t rowSize; SSqlGroupbyExpr* pGroupbyExpr; SSqlFunctionExpr* pSelectExpr; - SColumnInfoEx* colList; + SColumnInfoData* colList; int32_t numOfFilterCols; int64_t* defaultVal; TSKEY lastKey; diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 1ba74af2be..6e540f6ae1 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -866,7 +866,7 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 int32_t numOfCols = taosArrayGetSize(pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoEx *p = taosArrayGet(pDataBlock, i); + SColumnInfoData *p = taosArrayGet(pDataBlock, i); if (pCol->colId == p->info.colId) { dataBlock = p->pData; break; @@ -894,7 +894,7 @@ static void blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStati SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; SQuery * pQuery = pRuntimeEnv->pQuery; - SColumnInfoEx *pColInfo = NULL; + SColumnInfoData *pColInfo = NULL; TSKEY * primaryKeyCol = NULL; if (pDataBlock != NULL) { @@ -2223,7 +2223,7 @@ char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWi pQuery->pSelectExpr[columnIndex].resBytes * realRowId; } -int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { +int32_t UNUSED_FUNC vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { dTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, @@ -2292,11 +2292,12 @@ int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, true); - STsdbQueryCond cond = {0}; - cond.twindow = (STimeWindow){.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}; - cond.order = pQuery->order.order; - - cond.colList = *pQuery->colList; + STsdbQueryCond cond = { + .twindow = (STimeWindow) {.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}, + .order = pQuery->order.order, + .colList = pQuery->colList, + }; + SArray *sa = taosArrayInit(1, POINTER_BYTES); // for(int32_t i = 0; i < pQInfo->pSidSet->numOfTables; ++i) { @@ -2699,7 +2700,6 @@ void vnodeSetTagValueInParam(tSidSet *pSidSet, SQueryRuntimeEnv *pRuntimeEnv, vo } // set the join tag for first column - SSqlFuncExprMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; if (pFuncMsg->functionId == TSDB_FUNC_TS && pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX && pRuntimeEnv->pTSBuf != NULL) { assert(pFuncMsg->numOfParams == 1); @@ -3472,8 +3472,8 @@ void setQueryStatus(SQuery *pQuery, int8_t status) { bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - bool toContinue = false; - + + bool toContinue = false; if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { // for each group result, call the finalize function for each column SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; @@ -3567,8 +3567,7 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { doSingleMeterSupplementScan(pRuntimeEnv); - // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query during - // supplementary scan + // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query during reverse scan pQuery->lastKey = lkey; pQuery->window.ekey = ekey; @@ -3577,7 +3576,7 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { // tsdbNextDataBlock(pRuntimeEnv->pQueryHandle); } -void doFinalizeResult(SQueryRuntimeEnv *pRuntimeEnv) { +void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { @@ -4170,7 +4169,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) STsdbQueryCond cond = { .twindow = pQuery->window, .order = pQuery->order.order, - .colList = *pQuery->colList, + .colList = pQuery->colList, }; SArray *cols = taosArrayInit(pQuery->numOfCols, sizeof(pQuery->colList[0])); @@ -4450,7 +4449,7 @@ static UNUSED_FUNC int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, i scanAllDataBlocks(pRuntimeEnv); // first/last_row query, do not invoke the finalize for super table query - doFinalizeResult(pRuntimeEnv); + finalizeQueryResult(pRuntimeEnv); int64_t numOfRes = getNumOfResult(pRuntimeEnv); assert(numOfRes == 1 || numOfRes == 0); @@ -4690,7 +4689,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { * Only the ts-comp query requires the finalizer function to be executed here. */ if (isTSCompQuery(pQuery)) { - doFinalizeResult(pRuntimeEnv); + finalizeQueryResult(pRuntimeEnv); } if (pRuntimeEnv->pTSBuf != NULL) { @@ -4896,12 +4895,12 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { * select count(*)/top(field,k)/avg(field name) from table_name [where ts>now-1a]; * select count(*) from table_name group by status_column; */ -static void tableFixedOutputProcessor(SQInfo *pQInfo) { +static void tableFixedOutputProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; scanAllDataBlocks(pRuntimeEnv); - doFinalizeResult(pRuntimeEnv); + finalizeQueryResult(pRuntimeEnv); if (isQueryKilled(pQInfo)) { return; @@ -4917,8 +4916,6 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { doSkipResults(pRuntimeEnv); doRevisedResultsByLimit(pQInfo); - - pQuery->rec.rows = pQuery->rec.rows; } static void tableMultiOutputProcess(SQInfo *pQInfo) { @@ -4932,7 +4929,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo) { while (1) { scanAllDataBlocks(pRuntimeEnv); - doFinalizeResult(pRuntimeEnv); + finalizeQueryResult(pRuntimeEnv); if (isQueryKilled(pQInfo)) { return; @@ -4983,7 +4980,7 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { } assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_NOT_COMPLETED)); - doFinalizeResult(pRuntimeEnv); + finalizeQueryResult(pRuntimeEnv); // here we can ignore the records in case of no interpolation // todo handle offset, in case of top/bottom interval query @@ -5004,7 +5001,7 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { } // handle time interval query on table -static void tableIntervalProcessor(SQInfo *pQInfo) { +static void tableIntervalProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &(pQInfo->runtimeEnv); SQuery * pQuery = pRuntimeEnv->pQuery; @@ -5076,8 +5073,6 @@ static void tableQueryImpl(SQInfo* pQInfo) { doRevisedResultsByLimit(pQInfo); pQInfo->pointsInterpo += numOfInterpo; - pQuery->rec.rows += pQuery->rec.rows; - dTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); sem_post(&pQInfo->dataReady); return; @@ -5119,15 +5114,14 @@ static void tableQueryImpl(SQInfo* pQInfo) { // group by normal column, sliding window query, interval query are handled by interval query processor if (isIntervalQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) - tableIntervalProcessor(pQInfo); + tableIntervalProcess(pQInfo); } else if (isFixedOutputQuery(pQuery)) { - tableFixedOutputProcessor(pQInfo); + tableFixedOutputProcess(pQInfo); } else { // diff/add/multiply/subtract/division assert(pQuery->checkBuffer == 1); tableMultiOutputProcess(pQInfo); } - // record the total elapsed time pQInfo->elapsedTime += (taosGetTimestampUs() - st); assert(taosArrayGetSize(pQInfo->pTableIdList) == 1); @@ -5593,7 +5587,7 @@ static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { if (pQuery->colList[i].info.numOfFilters > 0) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[j]; - memcpy(&pFilterInfo->info, &pQuery->colList[i], sizeof(SColumnInfoEx)); + memcpy(&pFilterInfo->info, &pQuery->colList[i], sizeof(SColumnInfoData)); pFilterInfo->info.info.filters = NULL; pFilterInfo->numOfFilters = pQuery->colList[i].info.numOfFilters; diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h index ff3d4666fc..4f86066755 100644 --- a/src/vnode/tsdb/inc/tsdb.h +++ b/src/vnode/tsdb/inc/tsdb.h @@ -188,7 +188,7 @@ typedef void* tsdb_query_handle_t; // Use void to hide implementation details typedef struct STsdbQueryCond { STimeWindow twindow; int32_t order; // desc/asc order to iterate the data block - SColumnInfoEx colList; + SColumnInfoData* colList; } STsdbQueryCond; typedef struct SBlockInfo { diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 30181bed1d..e5f0aed05e 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -25,7 +25,7 @@ #include "tsdbMain.h" #define EXTRA_BYTES 2 -#define PRIMARY_TSCOL_REQUIRED(c) (((SColumnInfoEx*)taosArrayGet(c, 0))->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) +#define PRIMARY_TSCOL_REQUIRED(c) (((SColumnInfoData*)taosArrayGet(c, 0))->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) #define QUERY_IS_ASC_QUERY(o) (o == TSDB_ORDER_ASC) #define QH_GET_NUM_OF_COLS(handle) (taosArrayGetSize((handle)->pColumns)) @@ -65,7 +65,10 @@ typedef struct STableCheckInfo { int64_t offsetInHeaderFile; int32_t start; bool checkFirstFileBlock; + SCompInfo* pCompInfo; + int32_t compSize; + int32_t numOfBlocks; // number of qualified data blocks not the original blocks SDataCols* pDataCols; @@ -106,7 +109,7 @@ typedef struct STsdbQueryHandle { SCompBlock* pBlock; int32_t numOfBlocks; SField** pFields; - SArray* pColumns; // column list, SColumnInfoEx array list + SArray* pColumns; // column list, SColumnInfoData array list bool locateStart; int32_t realNumOfRows; bool loadDataAfterSeek; // load data after seek. @@ -127,7 +130,7 @@ typedef struct STsdbQueryHandle { int32_t doAllocateBuf(STsdbQueryHandle* pQueryHandle, int32_t rowsPerFileBlock) { // record the maximum column width among columns of this meter/metric - SColumnInfoEx* pColumn = taosArrayGet(pQueryHandle->pColumns, 0); + SColumnInfoData* pColumn = taosArrayGet(pQueryHandle->pColumns, 0); int32_t maxColWidth = pColumn->info.bytes; for (int32_t i = 1; i < QH_GET_NUM_OF_COLS(pQueryHandle); ++i) { @@ -177,7 +180,7 @@ tsdb_query_handle_t* tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond* pCond .lastKey = pQueryHandle->window.skey, .tableId = id, .pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id.uid), // todo this may be failed - .pCompInfo = calloc(1, 1024), + .pCompInfo = NULL, }; assert(info.pTableObj != NULL); @@ -193,10 +196,10 @@ tsdb_query_handle_t* tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond* pCond int32_t numOfCols = taosArrayGetSize(pColumnInfo); size_t bufferCapacity = 4096; - pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoEx)); + pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoEx* pCol = taosArrayGet(pColumnInfo, i); - SColumnInfoEx pDest = {{0}, 0}; + SColumnInfoData* pCol = taosArrayGet(pColumnInfo, i); + SColumnInfoData pDest = {{0}, 0}; pDest.pData = calloc(1, EXTRA_BYTES + bufferCapacity * pCol->info.bytes); pDest.info = pCol->info; @@ -261,6 +264,15 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file } else { + if (pCheckInfo->compSize < compIndex->len) { + assert(compIndex->len > 0); + + char* t = realloc(pCheckInfo->pCompInfo, compIndex->len); + assert(t != NULL); + + pCheckInfo->pCompInfo = (SCompInfo*) t; + } + tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo); int32_t index = 0; @@ -589,7 +601,7 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf int16_t colId = *(int16_t*)taosArrayGet(sa, i); for (int32_t j = 0; j < numOfCols; ++j) { - SColumnInfoEx* pCol = taosArrayGet(pQueryHandle->pColumns, j); + SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, j); if (pCol->info.colId == colId) { SDataCol* pDataCol = &pCols->cols[i]; @@ -610,7 +622,7 @@ static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) { int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoEx* pCol = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); taosArrayPush(pIdList, &pCol->info.colId); } @@ -1067,7 +1079,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int max int32_t offset = 0; for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoEx* pColInfo = taosArrayGet(pHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pHandle->pColumns, i); memcpy(pColInfo->pData + numOfRows * pColInfo->info.bytes, dataRowTuple(row) + offset, pColInfo->info.bytes); offset += pColInfo->info.bytes; } @@ -1102,7 +1114,7 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t* pQueryHandle) { /* not a whole disk block, only the qualified rows, so this block is loaded in to buffer during the * block next function */ - SColumnInfoEx* pColInfoEx = taosArrayGet(pHandle->pColumns, 0); + SColumnInfoData* pColInfoEx = taosArrayGet(pHandle->pColumns, 0); rows = pHandle->realNumOfRows; skey = *(TSKEY*)pColInfoEx->pData; @@ -1141,7 +1153,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t* pQueryHandle, SData SArray* tsdbRetrieveDataBlock(tsdb_query_handle_t* pQueryHandle, SArray* pIdList) { /** - * In the following two cases, the data has been loaded to SColumnInfoEx. + * In the following two cases, the data has been loaded to SColumnInfoData. * 1. data is from cache, 2. data block is not completed qualified to query time range */ STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; @@ -1484,7 +1496,7 @@ void tsdbCleanupQueryHandle(tsdb_query_handle_t queryHandle) { size_t cols = taosArrayGetSize(pQueryHandle->pColumns); for (int32_t i = 0; i < cols; ++i) { - SColumnInfoEx* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); tfree(pColInfo->pData); } From e7f46e4a1ca81a0e090e83a0eda617a74705b6ab Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 11 Apr 2020 15:13:08 +0800 Subject: [PATCH 52/78] add simplest python testcase for 2.0 [TD-128] --- .gitignore | 1 - .travis.yml | 46 ++++- tests/pytest/insert/__init__.py | 0 tests/pytest/insert/basic.py | 53 +++++ tests/pytest/simpletest.sh | 1 + tests/pytest/test.py | 87 +++++++++ tests/pytest/util/__init__.py | 0 tests/pytest/util/cases.py | 103 ++++++++++ tests/pytest/util/dnodes.py | 332 ++++++++++++++++++++++++++++++++ tests/pytest/util/log.py | 48 +++++ tests/pytest/util/sql.py | 135 +++++++++++++ 11 files changed, 801 insertions(+), 5 deletions(-) create mode 100644 tests/pytest/insert/__init__.py create mode 100644 tests/pytest/insert/basic.py create mode 100755 tests/pytest/simpletest.sh create mode 100644 tests/pytest/test.py create mode 100644 tests/pytest/util/__init__.py create mode 100644 tests/pytest/util/cases.py create mode 100644 tests/pytest/util/dnodes.py create mode 100644 tests/pytest/util/log.py create mode 100644 tests/pytest/util/sql.py diff --git a/.gitignore b/.gitignore index f2c1cb75b3..e91d6739a5 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,6 @@ tests/test/ tests/taoshebei/ tests/taoscsv/ tests/taosdalipu/ -tests/pytest/ tests/jenkins/ tests/hdfs/ *.iml diff --git a/.travis.yml b/.travis.yml index 63c62d1a8a..221a1f1a30 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,15 @@ os: - linux # - osx +before_install: + - |- + case $TRAVIS_OS_NAME in + linux) + sudo apt -y update + sudo apt -y install python-pip python3-pip python-setuptools python3-setuptools + ;; + esac + addons: coverity_scan: @@ -50,6 +59,15 @@ script: - |- case $TRAVIS_OS_NAME in linux) + # Color setting + RED='\033[0;31m' + GREEN='\033[1;32m' + GREEN_DARK='\033[0;32m' + GREEN_UNDERLINE='\033[4;32m' + NC='\033[0m' + + sudo make install + cd ../tests/script sudo ./test.sh 2>&1 | grep 'success\|failed' | tee out.txt @@ -57,16 +75,32 @@ script: if [ "$total_success" -gt "0" ]; then total_success=`expr $total_success - 1` + echo -e "${GREEN} ### Total $total_success TSIM case(s) succeed! ### ${NC}" fi - echo "Total $total_success success" - total_failed=`grep failed out.txt | wc -l` - echo "Total $total_failed failed" - if [ "$total_failed" -ne "0" ]; then + echo -e "${RED} ### Total $total_failed TSIM case(s) failed! ### ${NC}" exit $total_failed fi + + pip install --user ../../src/connector/python/linux/python2/ + pip3 install --user ../../src/connector/python/linux/python3/ + + cd ../pytest + sudo ./simpletest.sh 2>&1 | grep 'successfully executed\|failed' | tee pytest-out.txt + total_py_success=`grep 'successfully executed' pytest-out.txt | wc -l` + + if [ "$total_py_success" -gt "0" ]; then + echo -e "${GREEN} ### Total $total_py_success python case(s) succeed! ### ${NC}" + fi + + total_py_failed=`grep 'failed' pytest-out.txt | wc -l` + if [ "$total_py_failed" -ne "0" ]; then + echo -e "${RED} ### Total $total_py_failed python case(s) failed! ### ${NC}" + exit $total_py_failed + fi + ;; esac @@ -81,6 +115,10 @@ matrix: - build-essential - cmake - net-tools + - python-pip + - python-setuptools + - python3-pip + - python3-setuptools # - os: osx # addons: diff --git a/tests/pytest/insert/__init__.py b/tests/pytest/insert/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pytest/insert/basic.py b/tests/pytest/insert/basic.py new file mode 100644 index 0000000000..c6dbd76de4 --- /dev/null +++ b/tests/pytest/insert/basic.py @@ -0,0 +1,53 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + tdSql.execute('show databases') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + tdSql.execute('create table tb (ts timestamp, speed int)') + + insertRows = 10 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(0, insertRows): + tdSql.execute('insert into tb values (now + %dm, %d)' % (i, i)) + +# tdLog.info("insert earlier data") +# tdSql.execute('insert into tb values (now - 5m , 10)') +# tdSql.execute('insert into tb values (now - 6m , 10)') +# tdSql.execute('insert into tb values (now - 7m , 10)') +# tdSql.execute('insert into tb values (now - 8m , 10)') + +# tdSql.query("select * from tb") +# tdSql.checkRows(insertRows) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/simpletest.sh b/tests/pytest/simpletest.sh new file mode 100755 index 0000000000..aab36884f3 --- /dev/null +++ b/tests/pytest/simpletest.sh @@ -0,0 +1 @@ +sudo python2 ./test.py -f insert/basic.py diff --git a/tests/pytest/test.py b/tests/pytest/test.py new file mode 100644 index 0000000000..b88e444665 --- /dev/null +++ b/tests/pytest/test.py @@ -0,0 +1,87 @@ +#!/usr/bin/python +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### +# install pip +# pip install src/connector/python/linux/python2/ + +# -*- coding: utf-8 -*- +import sys +import getopt +from util.log import * +from util.dnodes import * +from util.cases import * + +import taos + +# add testcase here: +from insert.basic import * + +if __name__ == "__main__": + fileName = "all" + deployPath = "" + masterIp = "" + testCluster = False + opts, args = getopt.getopt(sys.argv[1:], 'f:p:m:sch', [ + 'file=', 'path=', 'master', 'stop', 'cluster', 'help']) + for key, value in opts: + if key in ['-h', '--help']: + tdLog.printNoPrefix( + 'A collection of test cases written using Python') + tdLog.printNoPrefix('-f Name of test case file written by Python') + tdLog.printNoPrefix('-p Deploy Path for Simulator') + tdLog.printNoPrefix('-m Master Ip for Simulator') + tdLog.printNoPrefix('-c Test Cluster Flag') + tdLog.printNoPrefix('-s stop All dnodes') + sys.exit(0) + if key in ['-f', '--file']: + fileName = value + if key in ['-p', '--path']: + deployPath = value + if key in ['-m', '--master']: + masterIp = value + if key in ['-c', '--cluster']: + testCluster = True + if key in ['-s', '--stop']: + cmd = "ps -ef|grep -w taosd | grep 'taosd' | grep -v grep | awk '{print $2}' && pkill -9 taosd" + os.system(cmd) + tdLog.exit('stop All dnodes') + + if masterIp == "": + tdDnodes.init(deployPath) + if testCluster: + tdLog.notice("Procedures for testing cluster") + if fileName == "all": + tdCases.runAllCluster() + else: + tdCases.runOneCluster(fileName) + else: + tdLog.notice("Procedures for testing self-deployment") + tdDnodes.stopAll() + tdDnodes.deploy(1) + tdDnodes.start(1) + conn = taos.connect( + host='192.168.0.1', + config=tdDnodes.getSimCfgPath()) + if fileName == "all": + tdCases.runAllLinux(conn) + else: + tdLog.info("CBD LN78: %s" % (fileName)) + tdCases.runOneLinux(conn, fileName) + conn.close() + else: + tdLog.notice("Procedures for tdengine deployed in %s" % (masterIp)) + conn = taos.connect(host=masterIp, config=tdDnodes.getSimCfgPath()) + if fileName == "all": + tdCases.runAllWindows(conn) + else: + tdCases.runOneWindows(conn, fileName) + conn.close() + diff --git a/tests/pytest/util/__init__.py b/tests/pytest/util/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py new file mode 100644 index 0000000000..320c9d974f --- /dev/null +++ b/tests/pytest/util/cases.py @@ -0,0 +1,103 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import time +import datetime +from util.log import * + + +class TDCase: + def __init__(self, name, case): + self.name = name + self.case = case + + +class TDCases: + def __init__(self): + self.linuxCases = [] + self.windowsCases = [] + self.clusterCases = [] + + def addWindows(self, name, case): + self.windowsCases.append(TDCase(name, case)) + + def addLinux(self, name, case): + self.linuxCases.append(TDCase(name, case)) + + def addCluster(self, name, case): + self.clusterCases.append(TDCase(name, case)) + + def runAllLinux(self, conn): + tdLog.notice("run total %d cases" % (len(self.linuxCases))) + for case in self.linuxCases: + case.case.init(conn) + case.case.run() + case.case.stop() + tdLog.notice("total %d cases executed" % (len(self.linuxCases))) + + def runOneLinux(self, conn, fileName): + tdLog.notice("run cases like %s" % (fileName)) + runNum = 0 + for case in self.linuxCases: + if case.name.find(fileName) != -1: + case.case.init(conn) + case.case.run() + case.case.stop() + time.sleep(5) + runNum += 1 + tdLog.notice("total %d cases executed" % (runNum)) + + def runAllWindows(self, conn): + tdLog.notice("run total %d cases" % (len(self.windowsCases))) + for case in self.windowsCases: + case.case.init(conn) + case.case.run() + case.case.stop() + tdLog.notice("total %d cases executed" % (len(self.windowsCases))) + + def runOneWindows(self, conn, fileName): + tdLog.notice("run cases like %s" % (fileName)) + runNum = 0 + for case in self.windowsCases: + if case.name.find(fileName) != -1: + case.case.init(conn) + case.case.run() + case.case.stop() + time.sleep(2) + runNum += 1 + tdLog.notice("total %d cases executed" % (runNum)) + + def runAllCluster(self): + tdLog.notice("run total %d cases" % (len(self.clusterCases))) + for case in self.clusterCases: + case.case.init() + case.case.run() + case.case.stop() + tdLog.notice("total %d cases executed" % (len(self.clusterCases))) + + def runOneCluster(self, fileName): + tdLog.notice("run cases like %s" % (fileName)) + runNum = 0 + for case in self.clusterCases: + if case.name.find(fileName) != -1: + case.case.init() + case.case.run() + case.case.stop() + time.sleep(2) + runNum += 1 + tdLog.notice("total %d cases executed" % (runNum)) + + +tdCases = TDCases() diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py new file mode 100644 index 0000000000..2be4f94802 --- /dev/null +++ b/tests/pytest/util/dnodes.py @@ -0,0 +1,332 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import os.path +from util.log import * + + +class TDSimClient: + def init(self, path): + self.path = path + + def getCfgDir(self): + return self.cfgDir + + def cfg(self, option, value): + cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def deploy(self): + self.logDir = "%s/sim/psim/log" % (self.path,) + self.cfgDir = "%s/sim/psim/cfg" % (self.path) + self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path) + + cmd = "rm -rf " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "rm -rf " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "touch " + self.cfgPath + if os.system(cmd) != 0: + tdLog.exit(cmd) + + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") + self.cfg("logDir", self.logDir) + self.cfg("numOfLogLines", "100000000") + self.cfg("numOfThreadsPerCore", "2.0") + self.cfg("locale", "en_US.UTF-8") + self.cfg("charset", "GBK") + self.cfg("asyncLog", "0") + self.cfg("anyIp", "0") + self.cfg("sdbDebugFlag", "135") + self.cfg("rpcDebugFlag", "135") + self.cfg("tmrDebugFlag", "131") + self.cfg("cDebugFlag", "135") + self.cfg("udebugFlag", "135") + self.cfg("jnidebugFlag", "135") + self.cfg("qdebugFlag", "135") + tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) + + +class TDDnode: + def __init__(self, index): + self.index = index + self.running = 0 + self.deployed = 0 + + def init(self, path): + self.path = path + + def deploy(self): + self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index) + self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index) + self.cfgDir = "%s/sim/dnode%d/cfg" % (self.path, self.index) + self.cfgPath = "%s/sim/dnode%d/cfg/taos.cfg" % (self.path, self.index) + + cmd = "rm -rf " + self.dataDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "rm -rf " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "rm -rf " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.dataDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "touch " + self.cfgPath + if os.system(cmd) != 0: + tdLog.exit(cmd) + + self.startIP() + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") + self.cfg("publicIp", "192.168.0.%d" % (self.index)) + self.cfg("internalIp", "192.168.0.%d" % (self.index)) + self.cfg("privateIp", "192.168.0.%d" % (self.index)) + self.cfg("dataDir", self.dataDir) + self.cfg("logDir", self.logDir) + self.cfg("numOfLogLines", "100000000") + self.cfg("mgmtEqualVnodeNum", "0") + self.cfg("clog", "1") + self.cfg("statusInterval", "1") + self.cfg("numOfTotalVnodes", "64") + self.cfg("numOfMPeers", "3") + self.cfg("numOfThreadsPerCore", "2.0") + self.cfg("monitor", "0") + self.cfg("maxVnodeConnections", "30000") + self.cfg("maxMgmtConnections", "30000") + self.cfg("maxMeterConnections", "30000") + self.cfg("maxShellConns", "30000") + self.cfg("locale", "en_US.UTF-8") + self.cfg("charset", "UTF-8") + self.cfg("asyncLog", "0") + self.cfg("anyIp", "0") + self.cfg("dDebugFlag", "135") + self.cfg("mDebugFlag", "135") + self.cfg("sdbDebugFlag", "135") + self.cfg("rpcDebugFlag", "135") + self.cfg("tmrDebugFlag", "131") + self.cfg("cDebugFlag", "135") + self.cfg("httpDebugFlag", "135") + self.cfg("monitorDebugFlag", "135") + self.cfg("udebugFlag", "135") + self.cfg("jnidebugFlag", "135") + self.cfg("qdebugFlag", "135") + self.deployed = 1 + tdLog.debug( + "dnode:%d is deployed and configured by %s" % + (self.index, self.cfgPath)) + + def start(self): + binPath = os.path.dirname(os.path.realpath(__file__)) + binPath = binPath + "/../../../debug/" + binPath = os.path.realpath(binPath) + binPath += "/build/bin/" + + if self.deployed == 0: + tdLog.exit("dnode:%d is not deployed" % (self.index)) + cmd = "nohup %staosd -c %s > /dev/null 2>&1 & " % ( + binPath, self.cfgDir) + print(cmd) + if os.system(cmd) != 0: + tdLog.exit(cmd) + self.running = 1 + tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) + + tdLog.debug("wait 2 seconds for the dnode:%d to start." % (self.index)) + time.sleep(2) + + def stop(self): + if self.running != 0: + cmd = "ps -ef|grep -w taosd | grep '%s' | grep -v grep | awk '{print $2}' && pkill -sigint taosd" % ( + self.cfgDir) + if os.system(cmd) != 0: + tdLog.exit(cmd) + tdLog.debug("dnode:%d is stopped by kill -SIGINT" % (self.index)) + tdLog.debug( + "wait 2 seconds for the dnode:%d to stop." % + (self.index)) + time.sleep(2) + + def forcestop(self): + if self.running != 0: + cmd = "ps -ef|grep -w taosd | grep '%s' | grep -v grep | awk '{print $2}' && pkill -sigkill taosd" % ( + self.cfgDir) + if os.system(cmd) != 0: + tdLog.exit(cmd) + tdLog.debug("dnode:%d is stopped by kill -9" % (self.index)) + tdLog.debug( + "wait 2 seconds for the dnode:%d to stop." % + (self.index)) + time.sleep(2) + + def startIP(self): + cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def stopIP(self): + cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % ( + self.index, self.index) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def cfg(self, option, value): + cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def getDnodeRootDir(self, index): + dnodeRootDir = "%s/sim/psim/dnode%d" % (self.path, index) + return dnodeRootDir + + def getDnodesRootDir(self): + dnodesRootDir = "%s/sim/psim" % (self.path) + return dnodesRootDir + + +class TDDnodes: + def __init__(self): + self.dnodes = [] + self.dnodes.append(TDDnode(1)) + self.dnodes.append(TDDnode(2)) + self.dnodes.append(TDDnode(3)) + self.dnodes.append(TDDnode(4)) + self.dnodes.append(TDDnode(5)) + self.dnodes.append(TDDnode(6)) + self.dnodes.append(TDDnode(7)) + self.dnodes.append(TDDnode(8)) + self.dnodes.append(TDDnode(9)) + self.dnodes.append(TDDnode(10)) + + def init(self, path): + cmd = "ps -ef|grep -w taosd | grep 'taosd' | grep -v grep | awk '{print $2}' && pkill -sigkill taosd" + os.system(cmd) + + binPath = os.path.dirname(os.path.realpath(__file__)) + binPath = binPath + "/../../../debug/" + tdLog.debug("binPath %s" % (binPath)) + binPath = os.path.realpath(binPath) + tdLog.debug("binPath real path %s" % (binPath)) + + # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath) + # tdLog.debug(cmd) + # os.system(cmd) + + # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + # tdLog.debug("execute %s" % (cmd)) + + # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + # tdLog.debug("execute %s" % (cmd)) + + if path == "": + # self.path = os.path.expanduser('~') + self.path = os.path.abspath(binPath + "../../") + else: + self.path = os.path.realpath(path) + + for i in range(len(self.dnodes)): + self.dnodes[i].init(self.path) + + self.sim = TDSimClient() + self.sim.init(self.path) + self.sim.deploy() + + def deploy(self, index): + self.check(index) + self.dnodes[index - 1].deploy() + + def cfg(self, index, option, value): + self.check(index) + self.dnodes[index - 1].cfg(option, value) + + def start(self, index): + self.check(index) + self.dnodes[index - 1].start() + + def stop(self, index): + self.check(index) + self.dnodes[index - 1].stop() + + def forcestop(self, index): + self.check(index) + self.dnodes[index - 1].forcestop() + + def startIP(self, index): + self.check(index) + self.dnodes[index - 1].startIP() + + def stopIP(self, index): + self.check(index) + self.dnodes[index - 1].stopIP() + + def check(self, index): + if index < 1 or index > 10: + tdLog.exit("index:%d should on a scale of [1, 10]" % (index)) + + def stopAll(self): + tdLog.debug("stop all dnodes") + for i in range(len(self.dnodes)): + self.dnodes[i].stop() + + cmd = "sudo systemctl stop taosd" + os.system(cmd) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + cmd = "ps -ef | grep -w taosd | grep 'dnode' | grep -v grep | awk '{print $2}' && sudo pkill -sigkill taosd" + os.system(cmd) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + + def getDnodesRootDir(self): + dnodesRootDir = "%s/sim" % (self.path) + return dnodesRootDir + + def getSimCfgPath(self): + return self.sim.getCfgDir() + + +tdDnodes = TDDnodes() diff --git a/tests/pytest/util/log.py b/tests/pytest/util/log.py new file mode 100644 index 0000000000..926e582448 --- /dev/null +++ b/tests/pytest/util/log.py @@ -0,0 +1,48 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import time +import datetime + + +class TDLog: + def __init__(self): + self.path = "" + + def info(self, info): + print "%s %s" % (datetime.datetime.now(), info) + + def sleep(self, sec): + print "%s sleep %d seconds" % (datetime.datetime.now(), sec) + time.sleep(sec) + + def debug(self, err): + print "\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err) + + def success(self, info): + print "\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info) + + def notice(self, err): + print "\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err) + + def exit(self, err): + print "\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err) + sys.exit(1) + + def printNoPrefix(self, info): + print "\033[1;36m%s\033[0m" % (info) + + +tdLog = TDLog() diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py new file mode 100644 index 0000000000..b4ac845bc8 --- /dev/null +++ b/tests/pytest/util/sql.py @@ -0,0 +1,135 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import time +import datetime +from util.log import * + + +class TDSql: + def __init__(self): + self.queryRows = 0 + self.queryCols = 0 + self.affectedRows = 0 + + def init(self, cursor): + self.cursor = cursor + + def close(self): + self.cursor.close() + + def prepare(self): + tdLog.info("prepare database:db") + self.cursor.execute('reset query cache') + self.cursor.execute('drop database if exists db') + self.cursor.execute('create database db') + self.cursor.execute('use db') + + def error(self, sql): + expectErrNotOccured = True + try: + self.cursor.execute(sql) + except BaseException: + expectErrNotOccured = False + if expectErrNotOccured: + tdLog.exit("sql:%.40s, expect error not occured" % (sql)) + else: + tdLog.info("sql:%.40s, expect error occured" % (sql)) + + def query(self, sql): + self.sql = sql + self.cursor.execute(sql) + self.queryResult = self.cursor.fetchall() + self.queryRows = len(self.queryResult) + self.queryCols = len(self.cursor.description) + # if self.queryRows == 1 and self.queryCols == 1: + # tdLog.info("sql:%s, rows:%d cols:%d data:%s" % (self.sql, self.queryRows, self.queryCols, self.queryResult[0][0])) + # else: + # tdLog.info("sql:%s, rows:%d cols:%d" % (self.sql, self.queryRows, self.queryCols)) + return self.queryRows + + def checkRows(self, expectRows): + if self.queryRows != expectRows: + tdLog.exit( + "sql:%.40s, queryRows:%d != expect:%d" % + (self.sql, self.queryRows, expectRows)) + tdLog.info("sql:%.40s, queryRows:%d == expect:%d" % + (self.sql, self.queryRows, expectRows)) + + def checkData(self, row, col, data): + if row < 0: + tdLog.exit( + "sql:%.40s, row:%d is smaller than zero" % + (self.sql, row)) + if col < 0: + tdLog.exit( + "sql:%.40s, col:%d is smaller than zero" % + (self.sql, col)) + if row >= self.queryRows: + tdLog.exit( + "sql:%.40s, row:%d is larger than queryRows:%d" % + (self.sql, row, self.queryRows)) + if col >= self.queryCols: + tdLog.exit( + "sql:%.40s, col:%d is larger than queryRows:%d" % + (self.sql, col, self.queryCols)) + if self.queryResult[row][col] != data: + tdLog.exit( + "sql:%.40s row:%d col:%d data:%s != expect:%s" % + (self.sql, row, col, self.queryResult[row][col], data)) + tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%d" % + (self.sql, row, col, self.queryResult[row][col], data)) + + def getData(self, row, col): + if row < 0: + tdLog.exit( + "sql:%.40s, row:%d is smaller than zero" % + (self.sql, row)) + if col < 0: + tdLog.exit( + "sql:%.40s, col:%d is smaller than zero" % + (self.sql, col)) + if row >= self.queryRows: + tdLog.exit( + "sql:%.40s, row:%d is larger than queryRows:%d" % + (self.sql, row, self.queryRows)) + if col >= self.queryCols: + tdLog.exit( + "sql:%.40s, col:%d is larger than queryRows:%d" % + (self.sql, col, self.queryCols)) + return self.queryResult[row][col] + + def executeTimes(self, sql, times): + for i in range(times): + try: + return self.cursor.execute(sql) + except BaseException: + time.sleep(1) + continue + + def execute(self, sql): + self.sql = sql + self.affectedRows = self.cursor.execute(sql) + return self.affectedRows + + def checkAffectedRows(self, expectAffectedRows): + if self.affectedRows != expectAffectedRows: + tdLog.exit("sql:%.40s, affectedRows:%d != expect:%d" % + (self.sql, self.affectedRows, expectAffectedRows)) + tdLog.info("sql:%.40s, affectedRows:%d == expect:%d" % + (self.sql, self.affectedRows, expectAffectedRows)) + + +tdSql = TDSql() From 32bdf91387ac27fdd6cb576d7409deae9cbdbe64 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 11 Apr 2020 16:30:08 +0800 Subject: [PATCH 53/78] add one more parameter in walCfg keep the wal file when it is closed --- src/vnode/main/src/vnodeMain.c | 1 + src/vnode/wal/src/walMain.c | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index 210063ecf5..ea7a003d3d 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -368,6 +368,7 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { if (strcmp(option[0], "wals") != 0) return TSDB_CODE_INVALID_FILE_FORMAT; if (wals == -1) return TSDB_CODE_INVALID_FILE_FORMAT; pVnode->walCfg.wals = (int8_t)wals; + pVnode->walCfg.keep = 0; int32_t arbitratorIp = -1; num = fscanf(fp, "%s %u", option[0], &arbitratorIp); diff --git a/src/vnode/wal/src/walMain.c b/src/vnode/wal/src/walMain.c index e799bfd63a..edca4e371c 100644 --- a/src/vnode/wal/src/walMain.c +++ b/src/vnode/wal/src/walMain.c @@ -36,6 +36,7 @@ typedef struct { int fd; + int keep; int level; int max; // maximum number of wal files uint32_t id; // increase continuously @@ -61,6 +62,7 @@ void *walOpen(const char *path, const SWalCfg *pCfg) { pWal->id = 0; pWal->num = 0; pWal->level = pCfg->commitLog; + pWal->keep = pCfg->keep; strcpy(pWal->path, path); pthread_mutex_init(&pWal->mutex, NULL); @@ -82,18 +84,21 @@ void *walOpen(const char *path, const SWalCfg *pCfg) { void walClose(void *handle) { if (handle == NULL) return; - SWal *pWal = handle; - + SWal *pWal = handle; close(pWal->fd); - // remove all files in the directory - for (int i=0; inum; ++i) { - sprintf(pWal->name, "%s/%s%d", pWal->path, walPrefix, pWal->id-i); - if (remove(pWal->name) <0) { - wError("wal:%s, failed to remove", pWal->name); - } else { - wTrace("wal:%s, it is removed", pWal->name); + if (pWal->keep == 0) { + // remove all files in the directory + for (int i=0; inum; ++i) { + sprintf(pWal->name, "%s/%s%d", pWal->path, walPrefix, pWal->id-i); + if (remove(pWal->name) <0) { + wError("wal:%s, failed to remove", pWal->name); + } else { + wTrace("wal:%s, it is removed", pWal->name); + } } + } else { + wTrace("wal:%s, it is closed and kept", pWal->name); } pthread_mutex_destroy(&pWal->mutex); From 0743d310d312982eee64a7b60d024a8f1d8b6179 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 11 Apr 2020 16:34:40 +0800 Subject: [PATCH 54/78] add keep in walCfg --- src/inc/twal.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/inc/twal.h b/src/inc/twal.h index e90e2e54ce..06dc2e881b 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -36,12 +36,13 @@ typedef struct { typedef struct { int8_t commitLog; // commitLog int8_t wals; // number of WAL files; + int8_t keep; // keep the wal file when closed } SWalCfg; typedef void* twalh; // WAL HANDLE typedef int (*FWalWrite)(void *ahandle, void *pHead, int type); -twalh walOpen(const char *path, const SWalCfg *pCfg); +twalh walOpen(const char *path, const SWalCfg *pCfg); void walClose(twalh); int walRenew(twalh); int walWrite(twalh, SWalHead *); From 71607ccb8bfb43e98b4b1d8e51759b1accdcd5e5 Mon Sep 17 00:00:00 2001 From: slguan Date: Sat, 11 Apr 2020 16:39:11 +0800 Subject: [PATCH 55/78] [TD-52] update status message --- src/dnode/inc/dnodeMClient.h | 1 + src/dnode/src/dnodeMClient.c | 227 ++++++++++++++++++------- src/inc/mnode.h | 2 + src/inc/mpeer.h | 24 +-- src/inc/taosmsg.h | 15 +- src/mnode/src/mgmtDb.c | 2 - src/mnode/src/mgmtDnode.c | 4 +- src/mnode/src/mgmtMnode.c | 135 +++++++++------ src/mnode/src/mgmtProfile.c | 9 +- src/mnode/src/mgmtShell.c | 2 +- src/{plugins/http => util}/inc/cJSON.h | 0 src/{plugins/http => util}/src/cJSON.c | 0 12 files changed, 281 insertions(+), 140 deletions(-) rename src/{plugins/http => util}/inc/cJSON.h (100%) rename src/{plugins/http => util}/src/cJSON.c (100%) diff --git a/src/dnode/inc/dnodeMClient.h b/src/dnode/inc/dnodeMClient.h index cab9ea9be4..ba63894631 100644 --- a/src/dnode/inc/dnodeMClient.h +++ b/src/dnode/inc/dnodeMClient.h @@ -24,6 +24,7 @@ int32_t dnodeInitMClient(); void dnodeCleanupMClient(); void dnodeSendMsgToMnode(SRpcMsg *rpcMsg); uint32_t dnodeGetMnodeMasteIp(); +void * dnodeGetMpeerInfos(); #ifdef __cplusplus } diff --git a/src/dnode/src/dnodeMClient.c b/src/dnode/src/dnodeMClient.c index 5630872318..c09583bd65 100644 --- a/src/dnode/src/dnodeMClient.c +++ b/src/dnode/src/dnodeMClient.c @@ -15,33 +15,46 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "cJSON.h" #include "taosmsg.h" #include "tlog.h" #include "trpc.h" #include "tutil.h" +#include "tsync.h" #include "dnode.h" #include "dnodeMClient.h" #include "dnodeModule.h" #include "dnodeMgmt.h" +#define MPEER_CONTENT_LEN 2000 + static bool dnodeReadMnodeIpList(); static void dnodeSaveMnodeIpList(); static void dnodeProcessRspFromMnode(SRpcMsg *pMsg); static void dnodeProcessStatusRsp(SRpcMsg *pMsg); static void (*tsDnodeProcessMgmtRspFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); static void *tsDnodeMClientRpc = NULL; -static SRpcIpSet tsDnodeMnodeIpList = {0}; +static SRpcIpSet tsMpeerIpList = {0}; +static SDMNodeInfos tsMpeerInfos = {0}; int32_t dnodeInitMClient() { if (!dnodeReadMnodeIpList()) { - dTrace("failed to read mnode iplist, set it from cfg file"); - memset(&tsDnodeMnodeIpList, 0, sizeof(SRpcIpSet)); - tsDnodeMnodeIpList.port = tsMnodeDnodePort; - tsDnodeMnodeIpList.numOfIps = 1; - tsDnodeMnodeIpList.ip[0] = inet_addr(tsMasterIp); + memset(&tsMpeerIpList, 0, sizeof(SRpcIpSet)); + memset(&tsMpeerInfos, 0, sizeof(SDMNodeInfos)); + tsMpeerIpList.port = tsMnodeDnodePort; + tsMpeerIpList.numOfIps = 1; + tsMpeerIpList.ip[0] = inet_addr(tsMasterIp); if (tsSecondIp[0]) { - tsDnodeMnodeIpList.numOfIps = 2; - tsDnodeMnodeIpList.ip[1] = inet_addr(tsSecondIp); + tsMpeerIpList.numOfIps = 2; + tsMpeerIpList.ip[1] = inet_addr(tsSecondIp); + } + } else { + SRpcIpSet mgmtIpSet = {0}; + tsMpeerIpList.inUse = tsMpeerInfos.inUse; + tsMpeerIpList.numOfIps = tsMpeerInfos.nodeNum; + tsMpeerIpList.port = tsMpeerInfos.nodeInfos[0].nodePort; + for (int32_t i = 0; i < tsMpeerInfos.nodeNum; i++) { + tsMpeerIpList.ip[i] = tsMpeerInfos.nodeInfos[i].nodeIp; } } @@ -96,23 +109,31 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { } SDMStatusRsp *pStatusRsp = pMsg->pCont; - if (pStatusRsp->ipList.numOfIps <= 0) { - dError("status msg is invalid, num of ips is %d", pStatusRsp->ipList.numOfIps); + SDMNodeInfos *mpeers = &pStatusRsp->mpeers; + if (mpeers->nodeNum <= 0) { + dError("status msg is invalid, num of ips is %d", mpeers->nodeNum); return; } - pStatusRsp->ipList.port = htons(pStatusRsp->ipList.port); - for (int32_t i = 0; i < pStatusRsp->ipList.numOfIps; ++i) { - pStatusRsp->ipList.ip[i] = htonl(pStatusRsp->ipList.ip[i]); + SRpcIpSet mgmtIpSet = {0}; + mgmtIpSet.inUse = mpeers->inUse; + mgmtIpSet.numOfIps = mpeers->nodeNum; + mgmtIpSet.port = htons(mpeers->nodeInfos[0].nodePort); + for (int32_t i = 0; i < mpeers->nodeNum; i++) { + mgmtIpSet.ip[i] = htonl(mpeers->nodeInfos[i].nodeIp); } - //dTrace("status msg is received, result:%s", tstrerror(pMsg->code)); - - if (memcmp(&(pStatusRsp->ipList), &tsDnodeMnodeIpList, sizeof(SRpcIpSet)) != 0) { - dPrint("mnode ip list is changed, numOfIps:%d inUse:%d", pStatusRsp->ipList.numOfIps, pStatusRsp->ipList.inUse); - memcpy(&tsDnodeMnodeIpList, &pStatusRsp->ipList, sizeof(SRpcIpSet)); - for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; ++i) { - dPrint("mnode index:%d ip:%s", i, taosIpStr(tsDnodeMnodeIpList.ip[i])); + if (memcmp(&mgmtIpSet, &tsMpeerIpList, sizeof(SRpcIpSet)) != 0) { + memcpy(&tsMpeerIpList, &mgmtIpSet, sizeof(SRpcIpSet)); + memcpy(&tsMpeerInfos, mpeers, sizeof(SDMNodeInfos)); + dPrint("mnode ip list is changed, numOfIps:%d inUse:%d", tsMpeerInfos.nodeNum, tsMpeerInfos.inUse); + for (int32_t i = 0; i < mpeers->nodeNum; i++) { + tsMpeerInfos.nodeInfos[i].nodeId = htonl(mpeers->nodeInfos[i].nodeId); + tsMpeerInfos.nodeInfos[i].nodeIp = htonl(mpeers->nodeInfos[i].nodeIp); + tsMpeerInfos.nodeInfos[i].nodePort = htons(mpeers->nodeInfos[i].nodePort); + dPrint("mnode:%d, ip:%s:%u name:%s", tsMpeerInfos.nodeInfos[i].nodeId, + taosIpStr(tsMpeerInfos.nodeInfos[i].nodeId), tsMpeerInfos.nodeInfos[i].nodePort, + tsMpeerInfos.nodeInfos[i].nodeName); } dnodeSaveMnodeIpList(); } @@ -129,70 +150,148 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) { if (tsDnodeMClientRpc) { - rpcSendRequest(tsDnodeMClientRpc, &tsDnodeMnodeIpList, rpcMsg); + rpcSendRequest(tsDnodeMClientRpc, &tsMpeerIpList, rpcMsg); } } static bool dnodeReadMnodeIpList() { char ipFile[TSDB_FILENAME_LEN] = {0}; - sprintf(ipFile, "%s/iplist", tsDnodeDir); - + sprintf(ipFile, "%s/mgmtIpList.json", tsDnodeDir); FILE *fp = fopen(ipFile, "r"); - if (!fp) return false; - - char option[32] = {0}; - int32_t value = 0; - int32_t num = 0; - - num = fscanf(fp, "%s %d", option, &value); - if (num != 2) return false; - if (strcmp(option, "inUse") != 0) return false; - tsDnodeMnodeIpList.inUse = (int8_t)value;; - - num = fscanf(fp, "%s %d", option, &value); - if (num != 2) return false; - if (strcmp(option, "numOfIps") != 0) return false; - tsDnodeMnodeIpList.numOfIps = (int8_t)value; - - num = fscanf(fp, "%s %d", option, &value); - if (num != 2) return false; - if (strcmp(option, "port") != 0) return false; - tsDnodeMnodeIpList.port = (uint16_t)value; - - for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; i++) { - num = fscanf(fp, "%s %d", option, &value); - if (num != 2) return false; - if (strncmp(option, "ip", 2) != 0) return false; - tsDnodeMnodeIpList.ip[i] = (uint32_t)value; + if (!fp) { + dTrace("failed to read mnode mgmtIpList.json, file not exist"); + return false; } - fclose(fp); - dPrint("read mnode iplist successed"); - for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; i++) { - dPrint("mnode index:%d ip:%s", i, taosIpStr(tsDnodeMnodeIpList.ip[i])); - } + bool ret = false; + int maxLen = 2000; + char *content = calloc(1, maxLen + 1); + int len = fread(content, 1, maxLen, fp); + if (len <= 0) { + free(content); + fclose(fp); + dError("failed to read mnode mgmtIpList.json, content is null"); + return false; + } - return true; + cJSON* root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read mnode mgmtIpList.json, invalid json format"); + goto PARSE_OVER; + } + + cJSON* inUse = cJSON_GetObjectItem(root, "inUse"); + if (!inUse || inUse->type != cJSON_Number) { + dError("failed to read mnode mgmtIpList.json, inUse not found"); + goto PARSE_OVER; + } + tsMpeerInfos.inUse = inUse->valueint; + + cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum"); + if (!nodeNum || nodeNum->type != cJSON_Number) { + dError("failed to read mnode mgmtIpList.json, nodeNum not found"); + goto PARSE_OVER; + } + tsMpeerInfos.nodeNum = nodeNum->valueint; + + cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); + if (!nodeInfos || nodeInfos->type != cJSON_Array) { + dError("failed to read mnode mgmtIpList.json, nodeInfos not found"); + goto PARSE_OVER; + } + + int size = cJSON_GetArraySize(nodeInfos); + if (size != tsMpeerInfos.nodeNum) { + dError("failed to read mnode mgmtIpList.json, nodeInfos size not matched"); + goto PARSE_OVER; + } + + for (int i = 0; i < size; ++i) { + cJSON* nodeInfo = cJSON_GetArrayItem(nodeInfos, i); + if (nodeInfo == NULL) continue; + + cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); + if (!nodeId || nodeId->type != cJSON_Number) { + dError("failed to read mnode mgmtIpList.json, nodeId not found"); + goto PARSE_OVER; + } + tsMpeerInfos.nodeInfos[i].nodeId = nodeId->valueint; + + cJSON *nodeIp = cJSON_GetObjectItem(nodeInfo, "nodeIp"); + if (!nodeIp || nodeIp->type != cJSON_String || nodeIp->valuestring == NULL) { + dError("failed to read mnode mgmtIpList.json, nodeIp not found"); + goto PARSE_OVER; + } + tsMpeerInfos.nodeInfos[i].nodeIp = inet_addr(nodeIp->valuestring); + + cJSON *nodePort = cJSON_GetObjectItem(nodeInfo, "nodePort"); + if (!nodePort || nodePort->type != cJSON_Number) { + dError("failed to read mnode mgmtIpList.json, nodePort not found"); + goto PARSE_OVER; + } + tsMpeerInfos.nodeInfos[i].nodePort = (uint16_t)nodePort->valueint; + + cJSON *nodeName = cJSON_GetObjectItem(nodeInfo, "nodeName"); + if (!nodeIp || nodeName->type != cJSON_String || nodeName->valuestring == NULL) { + dError("failed to read mnode mgmtIpList.json, nodeName not found"); + goto PARSE_OVER; + } + strncpy(tsMpeerInfos.nodeInfos[i].nodeName, nodeName->valuestring, TSDB_NODE_NAME_LEN); + } + + ret = true; + + dPrint("read mnode iplist successed, numOfIps:%d inUse:%d", tsMpeerInfos.nodeNum, tsMpeerInfos.inUse); + for (int32_t i = 0; i < tsMpeerInfos.nodeNum; i++) { + dPrint("mnode:%d, ip:%s:%u name:%s", tsMpeerInfos.nodeInfos[i].nodeId, + taosIpStr(tsMpeerInfos.nodeInfos[i].nodeId), tsMpeerInfos.nodeInfos[i].nodePort, + tsMpeerInfos.nodeInfos[i].nodeName); + } + +PARSE_OVER: + free(content); + fclose(fp); + return ret; } static void dnodeSaveMnodeIpList() { char ipFile[TSDB_FILENAME_LEN] = {0}; - sprintf(ipFile, "%s/iplist", tsDnodeDir); - + sprintf(ipFile, "%s/mgmtIpList.json", tsDnodeDir); FILE *fp = fopen(ipFile, "w"); if (!fp) return; - fprintf(fp, "inUse %d\n", tsDnodeMnodeIpList.inUse); - fprintf(fp, "numOfIps %d\n", tsDnodeMnodeIpList.numOfIps); - fprintf(fp, "port %u\n", tsDnodeMnodeIpList.port); - for (int32_t i = 0; i < tsDnodeMnodeIpList.numOfIps; i++) { - fprintf(fp, "ip%d %u\n", i, tsDnodeMnodeIpList.ip[i]); + int32_t len = 0; + int32_t maxLen = 2000; + char * content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsMpeerInfos.inUse); + len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsMpeerInfos.nodeNum); + len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); + for (int32_t i = 0; i < tsMpeerInfos.nodeNum; i++) { + len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsMpeerInfos.nodeInfos[i].nodeId); + len += snprintf(content + len, maxLen - len, " \"nodeIp\": \"%s\",\n", taosIpStr(tsMpeerInfos.nodeInfos[i].nodeIp)); + len += snprintf(content + len, maxLen - len, " \"nodePort\": %u,\n", tsMpeerInfos.nodeInfos[i].nodePort); + len += snprintf(content + len, maxLen - len, " \"nodeName\": \"%s\"\n", tsMpeerInfos.nodeInfos[i].nodeName); + if (i < tsMpeerInfos.nodeNum -1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }]\n"); + } } - + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); fclose(fp); + free(content); + dPrint("save mnode iplist successed"); } uint32_t dnodeGetMnodeMasteIp() { - return tsDnodeMnodeIpList.ip[0]; + return tsMpeerIpList.ip[tsMpeerIpList.inUse]; +} + +void* dnodeGetMpeerInfos() { + return &tsMpeerInfos; } \ No newline at end of file diff --git a/src/inc/mnode.h b/src/inc/mnode.h index b19e91a90e..f2c072453f 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -52,8 +52,10 @@ typedef struct _mnode_obj { int32_t refCount; int8_t role; int8_t status; + uint16_t port; uint32_t privateIp; uint32_t publicIp; + char mnodeName[TSDB_NODE_NAME_LEN + 1]; } SMnodeObj; typedef struct _dnode_obj { diff --git a/src/inc/mpeer.h b/src/inc/mpeer.h index 7007f19226..157ea40119 100644 --- a/src/inc/mpeer.h +++ b/src/inc/mpeer.h @@ -20,28 +20,30 @@ extern "C" { #endif -enum _TSDB_MN_STATUS { - TSDB_MN_STATUS_OFFLINE, - TSDB_MN_STATUS_UNSYNCED, - TSDB_MN_STATUS_SYNCING, - TSDB_MN_STATUS_SERVING -}; +struct _mnode_obj; -enum _TSDB_MN_ROLE { - TSDB_MN_ROLE_UNDECIDED, - TSDB_MN_ROLE_SLAVE, - TSDB_MN_ROLE_MASTER +enum _TAOS_MN_STATUS { + TAOS_MN_STATUS_OFFLINE, + TAOS_MN_STATUS_DROPPING, + TAOS_MN_STATUS_READY }; int32_t mpeerInit(); void mpeerCleanup(); +int32_t mpeerGetMnodesNum(); +void * mpeerGetNextMnode(void *pNode, struct _mnode_obj **pMnode); +void mpeerReleaseMnode(struct _mnode_obj *pMnode); bool mpeerInServerStatus(); bool mpeerIsMaster(); +bool mpeerCheckRedirect(); -bool mpeerCheckRedirect(void *handle); void mpeerGetPrivateIpList(SRpcIpSet *ipSet); void mpeerGetPublicIpList(SRpcIpSet *ipSet); +void mpeerGetMpeerInfos(void *mpeers); + +char * mpeerGetMnodeStatusStr(int32_t status); +char * mpeerGetMnodeRoleStr(int32_t role); #ifdef __cplusplus } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 09d306ffad..5378bf4bae 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -593,7 +593,20 @@ typedef struct { } SDMStatusMsg; typedef struct { - SRpcIpSet ipList; + int32_t nodeId; + uint32_t nodeIp; + uint16_t nodePort; + char nodeName[TSDB_NODE_NAME_LEN + 1]; +} SDMNodeInfo; + +typedef struct { + int8_t inUse; + int8_t nodeNum; + SDMNodeInfo nodeInfos[TSDB_MAX_MPEERS]; +} SDMNodeInfos; + +typedef struct { + SDMNodeInfos mpeers; SDnodeState dnodeState; SVnodeAccess vnodeAccess[]; } SDMStatusRsp; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 0f18de3253..b1931347a7 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -678,8 +678,6 @@ static int32_t mgmtSetDbDropping(SDbObj *pDb) { } static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { - if (mpeerCheckRedirect(pMsg->thandle)) return; - SCMCreateDbMsg *pCreate = pMsg->pCont; pCreate->maxSessions = htonl(pCreate->maxSessions); pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 1d32ded088..51ac4e842d 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -141,8 +141,6 @@ static void clusterProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) { } void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { - if (mpeerCheckRedirect(rpcMsg->handle)) return; - SDMStatusMsg *pStatus = rpcMsg->pCont; pStatus->dnodeId = htonl(pStatus->dnodeId); pStatus->privateIp = htonl(pStatus->privateIp); @@ -221,7 +219,7 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { return; } - mpeerGetPrivateIpList(&pRsp->ipList); + mpeerGetMpeerInfos(&pRsp->mpeers); pRsp->dnodeState.dnodeId = htonl(pDnode->dnodeId); pRsp->dnodeState.moduleStatus = htonl(pDnode->moduleStatus); diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index c349969019..82da454793 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -17,41 +17,36 @@ #include "os.h" #include "taoserror.h" #include "trpc.h" +#include "tsync.h" #include "mpeer.h" #include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtUser.h" -#ifndef _MPEER - -static SMnodeObj tsMnodeObj = {0}; +extern int32_t mpeerInitMnodes(); +extern void mpeerCleanupMnodes(); static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); -int32_t mpeerInit() { - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mgmtGetMnodeMeta); - mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mgmtRetrieveMnodes); +#ifndef _MPEER +static SMnodeObj tsMnodeObj = {0}; + +int32_t mpeerInitMnodes() { tsMnodeObj.mnodeId = 1; + tsMnodeObj.dnodeId = 1; tsMnodeObj.privateIp = inet_addr(tsPrivateIp); tsMnodeObj.publicIp = inet_addr(tsPublicIp); tsMnodeObj.createdTime = taosGetTimestampMs(); - tsMnodeObj.role = TSDB_MN_ROLE_MASTER; - tsMnodeObj.status = TSDB_MN_STATUS_SERVING; + tsMnodeObj.role = TAOS_SYNC_ROLE_MASTER; + tsMnodeObj.status = TAOS_MN_STATUS_READY; + tsMnodeObj.port = tsMnodeDnodePort; + sprintf(tsMnodeObj.mnodeName, "m%d", tsMnodeObj.mnodeId); return TSDB_CODE_SUCCESS; } -void mpeerCleanup() {} -bool mpeerInServerStatus() { return tsMnodeObj.status == TSDB_MN_STATUS_SERVING; } -bool mpeerIsMaster() { return tsMnodeObj.role == TSDB_MN_ROLE_MASTER; } -bool mpeerCheckRedirect(void *thandle) { return false; } - -static int32_t mgmtGetMnodesNum() { - return 1; -} - -static void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) { +void *mpeerGetNextMnode(void *pNode, SMnodeObj **pMnode) { if (*pMnode == NULL) { *pMnode = &tsMnodeObj; } else { @@ -61,22 +56,74 @@ static void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) { return *pMnode; } -char *taosGetMnodeStatusStr(int32_t mnodeStatus) { - switch (mnodeStatus) { - case TSDB_MN_STATUS_OFFLINE: return "offline"; - case TSDB_MN_STATUS_UNSYNCED: return "unsynced"; - case TSDB_MN_STATUS_SYNCING: return "syncing"; - case TSDB_MN_STATUS_SERVING: return "serving"; - default: return "undefined"; +void mpeerGetPrivateIpList(SRpcIpSet *ipSet) { + ipSet->inUse = 0; + ipSet->port = htons(tsMnodeDnodePort); + ipSet->numOfIps = 1; + ipSet->ip[0] = htonl(tsMnodeObj.privateIp); +} + +void mpeerGetPublicIpList(SRpcIpSet *ipSet) { + ipSet->inUse = 0; + ipSet->port = htons(tsMnodeDnodePort); + ipSet->numOfIps = 1; + ipSet->ip[0] = htonl(tsMnodeObj.publicIp); +} + +void mpeerGetMpeerInfos(void *param) { + SDMNodeInfos *mpeers = param; + mpeers->nodeNum = 1; + mpeers->nodeInfos[0].nodeId = htonl(tsMnodeObj.mnodeId); + mpeers->nodeInfos[0].nodeIp = htonl(tsMnodeObj.privateIp); + mpeers->nodeInfos[0].nodePort = htons(tsMnodeObj.port); + strcpy(mpeers->nodeInfos[0].nodeName, tsMnodeObj.mnodeName); +} + +void mpeerCleanupDnodes() {} +int32_t mpeerGetMnodesNum() { return 1; } +void mpeerReleaseMnode(struct _mnode_obj *pMnode) {} +bool mpeerInServerStatus() { return tsMnodeObj.status == TAOS_MN_STATUS_READY; } +bool mpeerIsMaster() { return tsMnodeObj.role == TAOS_SYNC_ROLE_MASTER; } +bool mpeerCheckRedirect() { return false; } + +#endif + +int32_t mpeerInit() { + mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mgmtGetMnodeMeta); + mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mgmtRetrieveMnodes); + + return mpeerInitMnodes(); +} + +void mpeerCleanup() { + mpeerCleanupDnodes(); +} + +char *mpeerGetMnodeStatusStr(int32_t status) { + switch (status) { + case TAOS_MN_STATUS_OFFLINE: + return "offline"; + case TAOS_MN_STATUS_DROPPING: + return "dropping"; + case TAOS_MN_STATUS_READY: + return "ready"; + default: + return "undefined"; } } -char *taosGetMnodeRoleStr(int32_t mnodeRole) { - switch (mnodeRole) { - case TSDB_MN_ROLE_UNDECIDED: return "undicided"; - case TSDB_MN_ROLE_SLAVE: return "slave"; - case TSDB_MN_ROLE_MASTER: return "master"; - default: return "undefined"; +char *mpeerGetMnodeRoleStr(int32_t role) { + switch (role) { + case TAOS_SYNC_ROLE_OFFLINE: + return "offline"; + case TAOS_SYNC_ROLE_UNSYNCED: + return "unsynced"; + case TAOS_SYNC_ROLE_SLAVE: + return "slave"; + case TAOS_SYNC_ROLE_MASTER: + return "master"; + default: + return "undefined"; } } @@ -133,7 +180,7 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; } - pShow->numOfRows = mgmtGetMnodesNum(); + pShow->numOfRows = mpeerGetMnodesNum(); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->pNode = NULL; mgmtReleaseUser(pUser); @@ -149,7 +196,7 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi char ipstr[32]; while (numOfRows < rows) { - pShow->pNode = mgmtGetNextMnode(pShow->pNode, (SMnodeObj **)&pMnode); + pShow->pNode = mpeerGetNextMnode(pShow->pNode, &pMnode); if (pMnode == NULL) break; cols = 0; @@ -173,11 +220,11 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, taosGetMnodeStatusStr(pMnode->status)); + strcpy(pWrite, mpeerGetMnodeStatusStr(pMnode->status)); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, taosGetMnodeRoleStr(pMnode->role)); + strcpy(pWrite, mpeerGetMnodeRoleStr(pMnode->role)); cols++; numOfRows++; @@ -186,20 +233,4 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi pShow->numOfReads += numOfRows; return numOfRows; -} - -void mpeerGetPrivateIpList(SRpcIpSet *ipSet) { - ipSet->inUse = 0; - ipSet->port = htons(tsMnodeDnodePort); - ipSet->numOfIps = 1; - ipSet->ip[0] = htonl(tsMnodeObj.privateIp); -} - -void mpeerGetPublicIpList(SRpcIpSet *ipSet) { - ipSet->inUse = 0; - ipSet->port = htons(tsMnodeDnodePort); - ipSet->numOfIps = 1; - ipSet->ip[0] = htonl(tsMnodeObj.publicIp); -} - -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/src/mnode/src/mgmtProfile.c b/src/mnode/src/mgmtProfile.c index 36481b81b0..f7dec4656b 100644 --- a/src/mnode/src/mgmtProfile.c +++ b/src/mnode/src/mgmtProfile.c @@ -681,8 +681,7 @@ int32_t mgmtRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; - if (mpeerCheckRedirect(pMsg->thandle)) return; - + SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { rpcRsp.code = TSDB_CODE_INVALID_USER; @@ -705,8 +704,7 @@ void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) { void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; - if (mpeerCheckRedirect(pMsg->thandle)) return; - + SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { rpcRsp.code = TSDB_CODE_INVALID_USER; @@ -729,8 +727,7 @@ void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) { void mgmtProcessKillConnectionMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; - if (mpeerCheckRedirect(pMsg->thandle)) return; - + SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { rpcRsp.code = TSDB_CODE_INVALID_USER; diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index eb20225e95..dbd7627d3f 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -142,7 +142,7 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { return; } - if (mpeerCheckRedirect(rpcMsg->handle)) { + if (mpeerCheckRedirect()) { // rpcSendRedirectRsp(rpcMsg->handle, mgmtGetMnodeIpListForRedirect()); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NO_MASTER); rpcFreeCont(rpcMsg->pCont); diff --git a/src/plugins/http/inc/cJSON.h b/src/util/inc/cJSON.h similarity index 100% rename from src/plugins/http/inc/cJSON.h rename to src/util/inc/cJSON.h diff --git a/src/plugins/http/src/cJSON.c b/src/util/src/cJSON.c similarity index 100% rename from src/plugins/http/src/cJSON.c rename to src/util/src/cJSON.c From 2ff928db59516617a0caa91a4e1a0007aa8f0089 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 11 Apr 2020 17:31:55 +0800 Subject: [PATCH 56/78] [td-98] fix bug in taos_fetch_block --- src/client/src/tscSql.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 1eee5d5e7f..41cfb77c5d 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -698,6 +698,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { } int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { +#if 0 SSqlObj *pSql = (SSqlObj *)res; SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; @@ -737,6 +738,10 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { } return nRows; +#endif + + (*rows) = taos_fetch_row(res); + return ((*rows) != NULL)? 1:0; } int taos_select_db(TAOS *taos, const char *db) { From a9be7a8553ab0a89c1d93d7146dfd7b0b73e66db Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 11 Apr 2020 22:28:58 +0800 Subject: [PATCH 57/78] exit nicely --- src/dnode/src/dnodeRead.c | 84 +++++++++++++++++++++++----------- src/dnode/src/dnodeWrite.c | 31 +++++++++---- src/rpc/src/rpcCache.c | 3 +- src/rpc/src/rpcClient.c | 4 +- src/rpc/src/rpcServer.c | 7 ++- src/rpc/src/rpcUdp.c | 4 +- src/util/src/ihash.c | 1 - src/util/src/tqueue.c | 17 +++++-- src/util/src/tsched.c | 2 + src/vnode/main/src/vnodeMain.c | 3 +- 10 files changed, 106 insertions(+), 50 deletions(-) diff --git a/src/dnode/src/dnodeRead.c b/src/dnode/src/dnodeRead.c index d4365dae10..f13c18f22a 100644 --- a/src/dnode/src/dnodeRead.c +++ b/src/dnode/src/dnodeRead.c @@ -32,27 +32,51 @@ typedef struct { SRpcMsg rpcMsg; } SReadMsg; +typedef struct { + pthread_t thread; // thread + int32_t workerId; // worker ID +} SReadWorker; + +typedef struct { + int32_t max; // max number of workers + int32_t min; // min number of workers + int32_t num; // current number of workers + SReadWorker *readWorker; +} SReadWorkerPool; + static void *dnodeProcessReadQueue(void *param); -static void dnodeHandleIdleReadWorker(); +static void dnodeHandleIdleReadWorker(SReadWorker *); // module global variable -static taos_qset readQset; -static int32_t threads; // number of query threads -static int32_t maxThreads; -static int32_t minThreads; +static SReadWorkerPool readPool; +static taos_qset readQset; int32_t dnodeInitRead() { readQset = taosOpenQset(); - minThreads = 3; - maxThreads = tsNumOfCores * tsNumOfThreadsPerCore; - if (maxThreads <= minThreads * 2) maxThreads = 2 * minThreads; + readPool.min = 2; + readPool.max = tsNumOfCores * tsNumOfThreadsPerCore; + if (readPool.max <= readPool.min * 2) readPool.max = 2 * readPool.min; + readPool.readWorker = (SReadWorker *) calloc(sizeof(SReadWorker), readPool.max); + + if (readPool.readWorker == NULL) return -1; + for (int i=0; i < readPool.max; ++i) { + SReadWorker *pWorker = readPool.readWorker + i; + pWorker->workerId = i; + } dPrint("dnode read is opened"); return 0; } void dnodeCleanupRead() { + + for (int i=0; i < readPool.max; ++i) { + SReadWorker *pWorker = readPool.readWorker + i; + if (pWorker->thread) + pthread_join(pWorker->thread, NULL); + } + taosCloseQset(readQset); dPrint("dnode read is closed"); } @@ -116,18 +140,25 @@ void *dnodeAllocateRqueue(void *pVnode) { taosAddIntoQset(readQset, queue, pVnode); // spawn a thread to process queue - if (threads < maxThreads) { - pthread_t thread; - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + if (readPool.num < readPool.max) { + do { + SReadWorker *pWorker = readPool.readWorker + readPool.num; - if (pthread_create(&thread, &thAttr, dnodeProcessReadQueue, readQset) != 0) { - dError("failed to create thread to process read queue, reason:%s", strerror(errno)); - } + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessReadQueue, pWorker) != 0) { + dError("failed to create thread to process read queue, reason:%s", strerror(errno)); + } + + pthread_attr_destroy(&thAttr); + readPool.num++; + dTrace("read worker:%d is launched, total:%d", pWorker->workerId, readPool.num); + } while (readPool.num < readPool.min); } - dTrace("pVnode:%p, queue:%p is allocated", pVnode, queue); + dTrace("pVnode:%p, read queue:%p is allocated", pVnode, queue); return queue; } @@ -167,14 +198,14 @@ void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) { } static void *dnodeProcessReadQueue(void *param) { - taos_qset qset = (taos_qset)param; - SReadMsg *pReadMsg; - int type; - void *pVnode; + SReadWorker *pWorker = param; + SReadMsg *pReadMsg; + int type; + void *pVnode; while (1) { - if (taosReadQitemFromQset(qset, &type, (void **)&pReadMsg, (void **)&pVnode) == 0) { - dnodeHandleIdleReadWorker(); + if (taosReadQitemFromQset(readQset, &type, (void **)&pReadMsg, &pVnode) == 0) { + dnodeHandleIdleReadWorker(pWorker); continue; } @@ -186,11 +217,12 @@ static void *dnodeProcessReadQueue(void *param) { return NULL; } -static void dnodeHandleIdleReadWorker() { +static void dnodeHandleIdleReadWorker(SReadWorker *pWorker) { int32_t num = taosGetQueueNumber(readQset); - if (num == 0 || (num <= minThreads && threads > minThreads)) { - threads--; + if (num == 0 || (num <= readPool.min && readPool.num > readPool.min)) { + readPool.num--; + dTrace("read worker:%d is released, total:%d", pWorker->workerId, readPool.num); pthread_exit(NULL); } else { usleep(100); diff --git a/src/dnode/src/dnodeWrite.c b/src/dnode/src/dnodeWrite.c index 3c598ca360..b56e0d8ad7 100644 --- a/src/dnode/src/dnodeWrite.c +++ b/src/dnode/src/dnodeWrite.c @@ -28,6 +28,7 @@ #include "vnode.h" typedef struct { + taos_qall qall; taos_qset qset; // queue set pthread_t thread; // thread int32_t workerId; // worker ID @@ -65,6 +66,14 @@ int32_t dnodeInitWrite() { } void dnodeCleanupWrite() { + + for (int32_t i = 0; i < wWorkerPool.max; ++i) { + SWriteWorker *pWorker = wWorkerPool.writeWorker + i; + if (pWorker->thread) { + pthread_join(pWorker->thread, NULL); + } + } + free(wWorkerPool.writeWorker); dPrint("dnode write is closed"); } @@ -113,6 +122,7 @@ void *dnodeAllocateWqueue(void *pVnode) { if (pWorker->qset == NULL) return NULL; taosAddIntoQset(pWorker->qset, queue, pVnode); + pWorker->qall = taosAllocateQall(); wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max; pthread_attr_t thAttr; @@ -122,13 +132,17 @@ void *dnodeAllocateWqueue(void *pVnode) { if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessWriteQueue, pWorker) != 0) { dError("failed to create thread to process read queue, reason:%s", strerror(errno)); taosCloseQset(pWorker->qset); + } else { + dTrace("write worker:%d is launched", pWorker->workerId); } + + pthread_attr_destroy(&thAttr); } else { taosAddIntoQset(pWorker->qset, queue, pVnode); wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max; } - dTrace("pVnode:%p, queue:%p is allocated", pVnode, queue); + dTrace("pVnode:%p, write queue:%p is allocated", pVnode, queue); return queue; } @@ -160,17 +174,14 @@ void dnodeSendRpcWriteRsp(void *pVnode, void *param, int32_t code) { static void *dnodeProcessWriteQueue(void *param) { SWriteWorker *pWorker = (SWriteWorker *)param; - taos_qall qall; SWriteMsg *pWrite; SWalHead *pHead; int32_t numOfMsgs; int type; void *pVnode, *item; - qall = taosAllocateQall(); - while (1) { - numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, qall, &pVnode); + numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, pWorker->qall, &pVnode); if (numOfMsgs <=0) { dnodeHandleIdleWorker(pWorker); // thread exit if no queues anymore continue; @@ -178,7 +189,7 @@ static void *dnodeProcessWriteQueue(void *param) { for (int32_t i = 0; i < numOfMsgs; ++i) { pWrite = NULL; - taosGetQitem(qall, &type, &item); + taosGetQitem(pWorker->qall, &type, &item); if (type == TAOS_QTYPE_RPC) { pWrite = (SWriteMsg *)item; pHead = (SWalHead *)(pWrite->pCont - sizeof(SWalHead)); @@ -196,9 +207,9 @@ static void *dnodeProcessWriteQueue(void *param) { walFsync(vnodeGetWal(pVnode)); // browse all items, and process them one by one - taosResetQitems(qall); + taosResetQitems(pWorker->qall); for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, &type, &item); + taosGetQitem(pWorker->qall, &type, &item); if (type == TAOS_QTYPE_RPC) { pWrite = (SWriteMsg *)item; dnodeSendRpcWriteRsp(pVnode, item, pWrite->rpcMsg.code); @@ -209,8 +220,6 @@ static void *dnodeProcessWriteQueue(void *param) { } } - taosFreeQall(qall); - return NULL; } @@ -221,8 +230,10 @@ static void dnodeHandleIdleWorker(SWriteWorker *pWorker) { usleep(1000); sched_yield(); } else { + taosFreeQall(pWorker->qall); taosCloseQset(pWorker->qset); pWorker->qset = NULL; + dTrace("write worker:%d is released", pWorker->workerId); pthread_exit(NULL); } } diff --git a/src/rpc/src/rpcCache.c b/src/rpc/src/rpcCache.c index a397f6f845..a4863ef61d 100644 --- a/src/rpc/src/rpcCache.c +++ b/src/rpc/src/rpcCache.c @@ -103,7 +103,8 @@ void rpcCloseConnCache(void *handle) { if (pCache->connHashMemPool) taosMemPoolCleanUp(pCache->connHashMemPool); tfree(pCache->connHashList); - tfree(pCache->count) + tfree(pCache->count); + tfree(pCache->lockedBy); pthread_mutex_unlock(&pCache->mutex); diff --git a/src/rpc/src/rpcClient.c b/src/rpc/src/rpcClient.c index b362b1ba44..264449bbb0 100644 --- a/src/rpc/src/rpcClient.c +++ b/src/rpc/src/rpcClient.c @@ -84,7 +84,9 @@ void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&(pTcp->thread), &thattr, taosReadTcpData, (void *)(pTcp)) != 0) { + int code = pthread_create(&(pTcp->thread), &thattr, taosReadTcpData, (void *)(pTcp)); + pthread_attr_destroy(&thattr); + if (code != 0) { tError("%s failed to create TCP read data thread, reason:%s", label, strerror(errno)); return NULL; } diff --git a/src/rpc/src/rpcServer.c b/src/rpc/src/rpcServer.c index 1aadabc5f7..538b3059e3 100644 --- a/src/rpc/src/rpcServer.c +++ b/src/rpc/src/rpcServer.c @@ -83,6 +83,9 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, } memset(pServerObj->pThreadObj, 0, sizeof(SThreadObj) * (size_t)numOfThreads); + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + pThreadObj = pServerObj->pThreadObj; for (i = 0; i < numOfThreads; ++i) { pThreadObj->processData = fp; @@ -105,8 +108,6 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, return NULL; } - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&(pThreadObj->thread), &thattr, (void *)taosProcessTcpData, (void *)(pThreadObj)) != 0) { tError("%s failed to create TCP process data thread, reason:%s", label, strerror(errno)); return NULL; @@ -116,8 +117,6 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, pThreadObj++; } - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj)) != 0) { tError("%s failed to create TCP accept thread, reason:%s", label, strerror(errno)); return NULL; diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 64a4df0e73..785288f5b9 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -146,10 +146,12 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v pConn->tmrCtrl = pSet->tmrCtrl; } - if (pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn) != 0) { + int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn); + if (code != 0) { tError("%s failed to create thread to process UDP data, reason:%s", label, strerror(errno)); taosCloseSocket(pConn->fd); taosCleanUpUdpConnection(pSet); + pthread_attr_destroy(&thAttr); return NULL; } diff --git a/src/util/src/ihash.c b/src/util/src/ihash.c index 30773ae8d9..2cfadad964 100644 --- a/src/util/src/ihash.c +++ b/src/util/src/ihash.c @@ -189,7 +189,6 @@ void taosCleanUpIntHash(void *handle) { free(pObj->hashList); } - memset(pObj, 0, sizeof(IHashObj)); free(pObj->lockedBy); free(pObj); } diff --git a/src/util/src/tqueue.c b/src/util/src/tqueue.c index 2cf94267f8..b9f1141d35 100644 --- a/src/util/src/tqueue.c +++ b/src/util/src/tqueue.c @@ -117,7 +117,7 @@ int taosWriteQitem(taos_queue param, int type, void *item) { queue->numOfItems++; if (queue->qset) atomic_add_fetch_32(&queue->qset->numOfItems, 1); - pTrace("item:%p is put into queue, type:%d items:%d", item, type, queue->numOfItems); + pTrace("item:%p is put into queue:%p, type:%d items:%d", item, queue, type, queue->numOfItems); pthread_mutex_unlock(&queue->mutex); @@ -297,14 +297,16 @@ int taosReadQitemFromQset(taos_qset param, int *type, void **pitem, void **phand STaosQset *qset = (STaosQset *)param; STaosQnode *pNode = NULL; int code = 0; + + pthread_mutex_lock(&qset->mutex); for(int i=0; inumOfQueues; ++i) { - pthread_mutex_lock(&qset->mutex); + //pthread_mutex_lock(&qset->mutex); if (qset->current == NULL) qset->current = qset->head; STaosQueue *queue = qset->current; if (queue) qset->current = queue->next; - pthread_mutex_unlock(&qset->mutex); + //pthread_mutex_unlock(&qset->mutex); if (queue == NULL) break; pthread_mutex_lock(&queue->mutex); @@ -326,6 +328,8 @@ int taosReadQitemFromQset(taos_qset param, int *type, void **pitem, void **phand if (pNode) break; } + pthread_mutex_unlock(&qset->mutex); + return code; } @@ -335,13 +339,15 @@ int taosReadAllQitemsFromQset(taos_qset param, taos_qall p2, void **phandle) { STaosQall *qall = (STaosQall *)p2; int code = 0; + pthread_mutex_lock(&qset->mutex); + for(int i=0; inumOfQueues; ++i) { - pthread_mutex_lock(&qset->mutex); + // pthread_mutex_lock(&qset->mutex); if (qset->current == NULL) qset->current = qset->head; queue = qset->current; if (queue) qset->current = queue->next; - pthread_mutex_unlock(&qset->mutex); + // pthread_mutex_unlock(&qset->mutex); if (queue == NULL) break; pthread_mutex_lock(&queue->mutex); @@ -365,6 +371,7 @@ int taosReadAllQitemsFromQset(taos_qset param, taos_qall p2, void **phandle) { if (code != 0) break; } + pthread_mutex_unlock(&qset->mutex); return code; } diff --git a/src/util/src/tsched.c b/src/util/src/tsched.c index 56d16eeb71..11735d91b4 100644 --- a/src/util/src/tsched.c +++ b/src/util/src/tsched.c @@ -94,10 +94,12 @@ void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) { } pTrace("%s scheduler is initialized, numOfThreads:%d", pSched->label, pSched->numOfThreads); + pthread_attr_destroy(&attr); return (void *)pSched; _error: + pthread_attr_destroy(&attr); taosCleanUpScheduler(pSched); return NULL; } diff --git a/src/vnode/main/src/vnodeMain.c b/src/vnode/main/src/vnodeMain.c index ea7a003d3d..4d77e007ad 100644 --- a/src/vnode/main/src/vnodeMain.c +++ b/src/vnode/main/src/vnodeMain.c @@ -224,10 +224,11 @@ void vnodeRelease(void *pVnodeRaw) { // remove the whole directory } - dTrace("pVnode:%p vgId:%d, vnode is released", pVnode, pVnode->vgId); free(pVnode); int32_t count = atomic_sub_fetch_32(&tsOpennedVnodes, 1); + dTrace("pVnode:%p vgId:%d, vnode is released, vnodes:%d", pVnode, vgId, count); + if (count <= 0) { taosCleanUpIntHash(tsDnodeVnodesHash); vnodeModuleInit = PTHREAD_ONCE_INIT; From 39a0443ee31274e54a6b7722f2fc060950518df8 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 11 Apr 2020 23:18:13 +0800 Subject: [PATCH 58/78] [td-98] fix bug in descending order query --- src/vnode/tsdb/inc/tsdbMain.h | 5 +- src/vnode/tsdb/src/tsdbFile.c | 4 +- src/vnode/tsdb/src/tsdbRead.c | 250 ++++++++++++++++++---------------- 3 files changed, 139 insertions(+), 120 deletions(-) diff --git a/src/vnode/tsdb/inc/tsdbMain.h b/src/vnode/tsdb/inc/tsdbMain.h index 06f62ea6f7..077bdf45c3 100644 --- a/src/vnode/tsdb/inc/tsdbMain.h +++ b/src/vnode/tsdb/inc/tsdbMain.h @@ -222,8 +222,9 @@ int tsdbOpenFile(SFile *pFile, int oflag); int tsdbCloseFile(SFile *pFile); SFileGroup *tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid); int tsdbRemoveFileGroup(STsdbFileH *pFile, int fid); -#define TSDB_FGROUP_ITER_FORWARD 0 -#define TSDB_FGROUP_ITER_BACKWARD 1 +#define TSDB_FGROUP_ITER_FORWARD TSDB_ORDER_ASC +#define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC + typedef struct { int numOfFGroups; SFileGroup *base; diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index 8bdfe63002..d025144ba9 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -154,7 +154,7 @@ void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) { } int flags = (pIter->direction == TSDB_FGROUP_ITER_FORWARD) ? TD_GE : TD_LE; - void *ptr = taosbsearch(&fid, pIter->base, sizeof(SFileGroup), pIter->numOfFGroups, compFGroupKey, flags); + void *ptr = taosbsearch(&fid, pIter->base, pIter->numOfFGroups, sizeof(SFileGroup), compFGroupKey, flags); if (ptr == NULL) { pIter->pFileGroup = NULL; } else { @@ -173,7 +173,7 @@ SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) { pIter->pFileGroup += 1; } } else { - if (pIter->pFileGroup - 1 == pIter->base) { + if (pIter->pFileGroup == pIter->base) { pIter->pFileGroup = NULL; } else { pIter->pFileGroup -= 1; diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index e5f0aed05e..2fce73e547 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -27,7 +27,7 @@ #define EXTRA_BYTES 2 #define PRIMARY_TSCOL_REQUIRED(c) (((SColumnInfoData*)taosArrayGet(c, 0))->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) #define QUERY_IS_ASC_QUERY(o) (o == TSDB_ORDER_ASC) -#define QH_GET_NUM_OF_COLS(handle) (taosArrayGetSize((handle)->pColumns)) +#define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) enum { QUERY_RANGE_LESS_EQUAL = 0, @@ -87,14 +87,8 @@ typedef struct STableBlockInfo { int32_t groupIdx; /* number of group is less than the total number of tables */ } STableBlockInfo; -enum { - SINGLE_TABLE_MODEL = 1, - MULTI_TABLE_MODEL = 2, -}; - typedef struct STsdbQueryHandle { STsdbRepo* pTsdb; - int8_t model; // access model, single table model or multi-table model SQueryFilePos cur; // current position SQueryFilePos start; // the start position, used for secondary/third iteration @@ -128,21 +122,6 @@ typedef struct STsdbQueryHandle { SCompIdx* compIndex; } STsdbQueryHandle; -int32_t doAllocateBuf(STsdbQueryHandle* pQueryHandle, int32_t rowsPerFileBlock) { - // record the maximum column width among columns of this meter/metric - SColumnInfoData* pColumn = taosArrayGet(pQueryHandle->pColumns, 0); - - int32_t maxColWidth = pColumn->info.bytes; - for (int32_t i = 1; i < QH_GET_NUM_OF_COLS(pQueryHandle); ++i) { - int32_t bytes = pColumn[i].info.bytes; - if (bytes > maxColWidth) { - maxColWidth = bytes; - } - } - - return TSDB_CODE_SUCCESS; -} - static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; pBlockLoadInfo->sid = -1; @@ -161,9 +140,9 @@ tsdb_query_handle_t* tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond* pCond // todo 2. add the reference count for each table that is involved in query STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); - pQueryHandle->order = pCond->order; + pQueryHandle->order = pCond->order; pQueryHandle->window = pCond->twindow; - pQueryHandle->pTsdb = tsdb; + pQueryHandle->pTsdb = tsdb; pQueryHandle->compIndex = calloc(10000, sizeof(SCompIdx)), pQueryHandle->loadDataAfterSeek = false; @@ -174,25 +153,33 @@ tsdb_query_handle_t* tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond* pCond pQueryHandle->pTableCheckInfo = taosArrayInit(size, sizeof(STableCheckInfo)); for (int32_t i = 0; i < size; ++i) { - STableId id = *(STableId*)taosArrayGet(idList, i); - + STableId id = *(STableId*) taosArrayGet(idList, i); + + STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), id.uid); + if (pTable == NULL) { + dError("%p failed to get table, error uid:%" PRIu64, pQueryHandle, id.uid); + continue; + } + STableCheckInfo info = { - .lastKey = pQueryHandle->window.skey, - .tableId = id, - .pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id.uid), // todo this may be failed - .pCompInfo = NULL, + .lastKey = pQueryHandle->window.skey, + .tableId = id, + .pTableObj = pTable, }; - assert(info.pTableObj != NULL); taosArrayPush(pQueryHandle->pTableCheckInfo, &info); } - pQueryHandle->model = (size > 1) ? MULTI_TABLE_MODEL : SINGLE_TABLE_MODEL; - pQueryHandle->checkFiles = 1; - + dTrace("%p total numOfTable:%d in query", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo)); + + /* + * For ascending timestamp order query, query starts from data files. In contrast, buffer will be checked in the first place + * in case of descending timestamp order query. + */ + pQueryHandle->checkFiles = QUERY_IS_ASC_QUERY(pQueryHandle->order); pQueryHandle->activeIndex = 0; - // malloc buffer in order to load data from file + // allocate buffer in order to load data blocks from file int32_t numOfCols = taosArrayGetSize(pColumnInfo); size_t bufferCapacity = 4096; @@ -206,10 +193,6 @@ tsdb_query_handle_t* tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond* pCond taosArrayPush(pQueryHandle->pColumns, &pDest); } - if (doAllocateBuf(pQueryHandle, bufferCapacity) != TSDB_CODE_SUCCESS) { - return NULL; - } - tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); @@ -239,7 +222,12 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { // todo dynamic get the daysperfile static int32_t getFileIdFromKey(TSKEY key) { - return (int32_t)(key / 10); // set the starting fileId + int64_t fid = (int64_t)(key / 10); // set the starting fileId + if (fid > INT32_MAX) { + fid = INT32_MAX; + } + + return fid; } static int32_t binarySearchForBlockImpl(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order); @@ -247,8 +235,12 @@ static int32_t binarySearchForBlockImpl(SCompBlock* pBlock, int32_t numOfBlocks, static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks, int32_t type) { // todo check open file failed SFileGroup* fileGroup = pQueryHandle->pFileGroup; + + assert(fileGroup->files[TSDB_FILE_TYPE_HEAD].fname > 0); if (fileGroup->files[TSDB_FILE_TYPE_HEAD].fd == FD_INITIALIZER) { fileGroup->files[TSDB_FILE_TYPE_HEAD].fd = open(fileGroup->files[TSDB_FILE_TYPE_HEAD].fname, O_RDONLY); + } else { + assert(FD_VALID(fileGroup->files[TSDB_FILE_TYPE_HEAD].fd)); } // load all the comp offset value for all tables in this file @@ -262,7 +254,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo SCompIdx* compIndex = &pQueryHandle->compIndex[pCheckInfo->tableId.tid]; if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file - + assert(0); } else { if (pCheckInfo->compSize < compIndex->len) { assert(compIndex->len > 0); @@ -271,46 +263,35 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo assert(t != NULL); pCheckInfo->pCompInfo = (SCompInfo*) t; + pCheckInfo->compSize = compIndex->len; } tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo); - - int32_t index = 0; + + SCompInfo* pCompInfo = pCheckInfo->pCompInfo; + + TSKEY s = MIN(pCheckInfo->lastKey, pQueryHandle->window.ekey); + TSKEY e = MAX(pCheckInfo->lastKey, pQueryHandle->window.ekey); + // discard the unqualified data block based on the query time window - int32_t start = binarySearchForBlockImpl(pCheckInfo->pCompInfo->blocks, compIndex->numOfSuperBlocks, - pQueryHandle->order, pCheckInfo->lastKey); - - if (type == QUERY_RANGE_GREATER_EQUAL) { - if (pCheckInfo->lastKey <= pCheckInfo->pCompInfo->blocks[start].keyLast) { - // break; - } else { - index = -1; - } - } else { - if (pCheckInfo->lastKey >= pCheckInfo->pCompInfo->blocks[start].keyFirst) { - // break; - } else { - index = -1; - } - } - - // not found in data blocks in current file - if (index == -1) { + int32_t start = binarySearchForBlockImpl(pCompInfo->blocks, compIndex->numOfSuperBlocks, s, TSDB_ORDER_ASC); + int32_t end = start; + + if (s > pCompInfo->blocks[start].keyLast) { continue; } - // todo speedup the procedure of locating end block - int32_t e = start; - while (e < compIndex->numOfSuperBlocks && - (pCheckInfo->pCompInfo->blocks[e].keyFirst <= pQueryHandle->window.ekey)) { - e += 1; + // todo speedup the procedure of located end block + while (end < compIndex->numOfSuperBlocks && (pCompInfo->blocks[end].keyFirst <= e)) { + end += 1; } + pCheckInfo->numOfBlocks = (end - start); + if (start > 0) { - memmove(pCheckInfo->pCompInfo->blocks, &pCheckInfo->pCompInfo->blocks[start], (e - start) * sizeof(SCompBlock)); + memmove(pCompInfo->blocks, &pCompInfo->blocks[start], pCheckInfo->numOfBlocks * sizeof(SCompBlock)); } - pCheckInfo->numOfBlocks = (e - start); (*numOfBlocks) += pCheckInfo->numOfBlocks; } } @@ -413,7 +394,6 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock } SDataCols* pDataCols = pCheckInfo->pDataCols; - if (pCheckInfo->lastKey > pBlock->keyFirst) { cur->pos = binarySearchForKey(pDataCols->cols[0].pData, pBlock->numOfPoints, pCheckInfo->lastKey, pQueryHandle->order); @@ -425,8 +405,24 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock } else { // the whole block is loaded in to buffer pQueryHandle->realNumOfRows = pBlock->numOfPoints; } - } else { // todo desc query + } else { + // query ended in current block if (pQueryHandle->window.ekey > pBlock->keyFirst) { + if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { + return false; + } + + SDataCols* pDataCols = pCheckInfo->pDataCols; + if (pCheckInfo->lastKey < pBlock->keyLast) { + cur->pos = + binarySearchForKey(pDataCols->cols[0].pData, pBlock->numOfPoints, pCheckInfo->lastKey, pQueryHandle->order); + } else { + cur->pos = pBlock->numOfPoints - 1; + } + + filterDataInDataBlock(pQueryHandle, pCheckInfo, pBlock, sa); + } else { + pQueryHandle->realNumOfRows = pBlock->numOfPoints; } } @@ -619,7 +615,9 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf } static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) { - int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); + size_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); + assert(numOfCols <= TSDB_MAX_COLUMNS); + SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); @@ -923,8 +921,8 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { pQueryHandle->locateStart = true; int32_t fid = getFileIdFromKey(pQueryHandle->window.skey); - - tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, TSDB_FGROUP_ITER_FORWARD); + + tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order); tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); int32_t numOfBlocks = -1; @@ -934,13 +932,14 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { - if (getFileCompInfo(pQueryHandle, &numOfBlocks, 1) != TSDB_CODE_SUCCESS) { + int32_t type = QUERY_IS_ASC_QUERY(pQueryHandle->order)? QUERY_RANGE_GREATER_EQUAL:QUERY_RANGE_LESS_EQUAL; + if (getFileCompInfo(pQueryHandle, &numOfBlocks, type) != TSDB_CODE_SUCCESS) { break; } assert(numOfBlocks >= 0); dTrace("%p %d blocks found in file for %d table(s), fid:%d", pQueryHandle, numOfBlocks, - pQueryHandle->pFileGroup->fileId, numOfTables); + numOfTables, pQueryHandle->pFileGroup->fileId); // todo return error code to query engine if (createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks) != TSDB_CODE_SUCCESS) { @@ -961,7 +960,7 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { return false; } - cur->slot = 0; + cur->slot = QUERY_IS_ASC_QUERY(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; cur->fid = pQueryHandle->pFileGroup->fileId; STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; @@ -970,8 +969,10 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); } else { - if (cur->slot == pQueryHandle->numOfBlocks - 1) { // all blocks + if ((cur->slot == pQueryHandle->numOfBlocks - 1 && QUERY_IS_ASC_QUERY(pQueryHandle->order)) || + (cur->slot == 0 && !QUERY_IS_ASC_QUERY(pQueryHandle->order))) { // all blocks int32_t numOfBlocks = -1; + int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); pQueryHandle->numOfBlocks = 0; @@ -1001,67 +1002,84 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { cur->fid = -1; return false; } - - cur->slot = 0; + + cur->slot = QUERY_IS_ASC_QUERY(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; cur->fid = pQueryHandle->pFileGroup->fileId; STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; - STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; SCompBlock* pBlock = pBlockInfo->pBlock.compBlock; return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); } else { // next block of the same file - cur->slot += 1; - cur->pos = 0; - + int32_t step = QUERY_IS_ASC_QUERY(pQueryHandle->order)? 1:-1; + cur->slot += step; + STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; + if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { + cur->pos = 0; + } else { + cur->pos = pBlockInfo->pBlock.compBlock->numOfPoints - 1; + } + return loadFileDataBlock(pQueryHandle, pBlockInfo->pBlock.compBlock, pBlockInfo->pTableCheckInfo); } } } -// handle data in cache situation -bool tsdbNextDataBlock(tsdb_query_handle_t* pQueryHandle) { - STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; - - size_t numOfTables = taosArrayGetSize(pHandle->pTableCheckInfo); + +static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + // todo add assert, the value of numOfTables should be less than the maximum value for each vnode capacity assert(numOfTables > 0); - if (pHandle->checkFiles) { - if (getDataBlocksInFiles(pHandle)) { + while (pQueryHandle->activeIndex < numOfTables) { + if (hasMoreDataInCache(pQueryHandle)) { return true; } - pHandle->activeIndex = 0; - pHandle->checkFiles = 0; - - while (pHandle->activeIndex < numOfTables) { - if (hasMoreDataInCache(pHandle)) { - return true; - } - - pHandle->activeIndex += 1; - } - - return false; - } else { - while (pHandle->activeIndex < numOfTables) { - if (hasMoreDataInCache(pHandle)) { - return true; - } - - pHandle->activeIndex += 1; - } - - return false; + pQueryHandle->activeIndex += 1; } + + return false; +} + +// handle data in cache situation +bool tsdbNextDataBlock(tsdb_query_handle_t* pqHandle) { + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pqHandle; + + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + assert(numOfTables > 0); + + if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { + if (pQueryHandle->checkFiles) { + if (getDataBlocksInFiles(pQueryHandle)) { + return true; + } + + pQueryHandle->activeIndex = 0; + pQueryHandle->checkFiles = false; + } + + return doHasDataInBuffer(pQueryHandle); + } else { // starts from the buffer in case of descending timestamp order check data blocks + if (!pQueryHandle->checkFiles) { + if (doHasDataInBuffer(pQueryHandle)) { + return true; + } + + pQueryHandle->checkFiles = true; + } + + return getDataBlocksInFiles(pQueryHandle); + } + } static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey, - STsdbQueryHandle* pHandle) { + STsdbQueryHandle* pQueryHandle) { int numOfRows = 0; - int32_t numOfCols = taosArrayGetSize(pHandle->pColumns); + int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns); *skey = INT64_MIN; while (tSkipListIterNext(pIter)) { @@ -1079,7 +1097,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int max int32_t offset = 0; for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); memcpy(pColInfo->pData + numOfRows * pColInfo->info.bytes, dataRowTuple(row) + offset, pColInfo->info.bytes); offset += pColInfo->info.bytes; } From ea497f20a3de943c03e017d999df0ac3ca7fb312 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 11 Apr 2020 23:37:50 +0800 Subject: [PATCH 59/78] [td-98] fix uninitialized value access warning by valgrind. --- src/kit/shell/src/shellEngine.c | 2 +- src/kit/shell/src/shellMain.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 0325e7f641..e291fdc551 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -284,7 +284,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { /* Function to do regular expression check */ int regex_match(const char *s, const char *reg, int cflags) { regex_t regex; - char msgbuf[100]; + char msgbuf[100] = {0}; /* Compile regular expression */ if (regcomp(®ex, reg, cflags) != 0) { diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index a7b7e8383b..81a41453e8 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -97,6 +97,8 @@ int main(int argc, char* argv[]) { /* Interupt handler. */ struct sigaction act; + memset(&act, 0, sizeof(struct sigaction)); + act.sa_handler = interruptHandler; sigaction(SIGTERM, &act, NULL); sigaction(SIGINT, &act, NULL); From e4956e3f03452ccbb76e54f5039dc80307cd30e2 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 12 Apr 2020 10:54:10 +0800 Subject: [PATCH 60/78] remove invalid read in hash.c remove goto in tsched.c --- src/util/src/hash.c | 2 +- src/util/src/tsched.c | 111 ++++++++++++++++++++++-------------------- 2 files changed, 58 insertions(+), 55 deletions(-) diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 9cad14e8c7..03e155845e 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -298,7 +298,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool threadsafe) { */ static SHashNode *doCreateHashNode(const char *key, size_t keyLen, const char *pData, size_t dataSize, uint32_t hashVal) { - size_t totalSize = dataSize + sizeof(SHashNode) + keyLen; + size_t totalSize = dataSize + sizeof(SHashNode) + keyLen + 1; // one extra byte for null SHashNode *pNewNode = calloc(1, totalSize); if (pNewNode == NULL) { diff --git a/src/util/src/tsched.c b/src/util/src/tsched.c index 11735d91b4..8608b64057 100644 --- a/src/util/src/tsched.c +++ b/src/util/src/tsched.c @@ -40,68 +40,68 @@ static void *taosProcessSchedQueue(void *param); static void taosDumpSchedulerStatus(void *qhandle, void *tmrId); void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) { - pthread_attr_t attr; - SSchedQueue * pSched = (SSchedQueue *)malloc(sizeof(SSchedQueue)); + SSchedQueue *pSched = (SSchedQueue *)calloc(sizeof(SSchedQueue), 1); if (pSched == NULL) { - pError("%s: no enough memory for pSched, reason: %s", label, strerror(errno)); - goto _error; + pError("%s: no enough memory for pSched", label); + return NULL; + } + + pSched->queue = (SSchedMsg *)calloc(sizeof(SSchedMsg), queueSize); + if (pSched->queue == NULL) { + pError("%s: no enough memory for queue", label); + taosCleanUpScheduler(pSched); + return NULL; + } + + pSched->qthread = calloc(sizeof(pthread_t), numOfThreads); + if (pSched->qthread == NULL) { + pError("%s: no enough memory for qthread", label); + taosCleanUpScheduler(pSched); + return NULL; } - memset(pSched, 0, sizeof(SSchedQueue)); pSched->queueSize = queueSize; strncpy(pSched->label, label, sizeof(pSched->label)); // fix buffer overflow pSched->label[sizeof(pSched->label)-1] = '\0'; - if (pthread_mutex_init(&pSched->queueMutex, NULL) < 0) { - pError("init %s:queueMutex failed, reason:%s", pSched->label, strerror(errno)); - goto _error; - } - - if (tsem_init(&pSched->emptySem, 0, (unsigned int)pSched->queueSize) != 0) { - pError("init %s:empty semaphore failed, reason:%s", pSched->label, strerror(errno)); - goto _error; - } - - if (tsem_init(&pSched->fullSem, 0, 0) != 0) { - pError("init %s:full semaphore failed, reason:%s", pSched->label, strerror(errno)); - goto _error; - } - - if ((pSched->queue = (SSchedMsg *)malloc((size_t)pSched->queueSize * sizeof(SSchedMsg))) == NULL) { - pError("%s: no enough memory for queue, reason:%s", pSched->label, strerror(errno)); - goto _error; - } - - memset(pSched->queue, 0, (size_t)pSched->queueSize * sizeof(SSchedMsg)); pSched->fullSlot = 0; pSched->emptySlot = 0; - pSched->qthread = malloc(sizeof(pthread_t) * (size_t)numOfThreads); - if (pSched->qthread == NULL) { - pError("%s: no enough memory for qthread, reason: %s", pSched->label, strerror(errno)); - goto _error; + if (pthread_mutex_init(&pSched->queueMutex, NULL) < 0) { + pError("init %s:queueMutex failed(%s)", label, strerror(errno)); + taosCleanUpScheduler(pSched); + return NULL; } - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); + if (tsem_init(&pSched->emptySem, 0, (unsigned int)pSched->queueSize) != 0) { + pError("init %s:empty semaphore failed(%s)", label, strerror(errno)); + taosCleanUpScheduler(pSched); + return NULL; + } + + if (tsem_init(&pSched->fullSem, 0, 0) != 0) { + pError("init %s:full semaphore failed(%s)", label, strerror(errno)); + taosCleanUpScheduler(pSched); + return NULL; + } for (int i = 0; i < numOfThreads; ++i) { - if (pthread_create(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched) != 0) { - pError("%s: failed to create rpc thread, reason:%s", pSched->label, strerror(errno)); - goto _error; + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); + int code = pthread_create(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched); + pthread_attr_destroy(&attr); + if (code != 0) { + pError("%s: failed to create rpc thread(%s)", label, strerror(errno)); + taosCleanUpScheduler(pSched); + return NULL; } ++pSched->numOfThreads; } - pTrace("%s scheduler is initialized, numOfThreads:%d", pSched->label, pSched->numOfThreads); - pthread_attr_destroy(&attr); + pTrace("%s scheduler is initialized, numOfThreads:%d", label, pSched->numOfThreads); return (void *)pSched; - -_error: - pthread_attr_destroy(&attr); - taosCleanUpScheduler(pSched); - return NULL; } void *taosInitSchedulerWithInfo(int queueSize, int numOfThreads, const char *label, void *tmrCtrl) { @@ -126,21 +126,21 @@ void *taosProcessSchedQueue(void *param) { pTrace("wait %s fullSem was interrupted", pSched->label); continue; } - pError("wait %s fullSem failed, errno:%d, reason:%s", pSched->label, errno, strerror(errno)); + pError("wait %s fullSem failed(%s)", pSched->label, strerror(errno)); } if (pthread_mutex_lock(&pSched->queueMutex) != 0) - pError("lock %s queueMutex failed, reason:%s", pSched->label, strerror(errno)); + pError("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); msg = pSched->queue[pSched->fullSlot]; memset(pSched->queue + pSched->fullSlot, 0, sizeof(SSchedMsg)); pSched->fullSlot = (pSched->fullSlot + 1) % pSched->queueSize; if (pthread_mutex_unlock(&pSched->queueMutex) != 0) - pError("unlock %s queueMutex failed, reason:%s\n", pSched->label, strerror(errno)); + pError("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); if (tsem_post(&pSched->emptySem) != 0) - pError("post %s emptySem failed, reason:%s\n", pSched->label, strerror(errno)); + pError("post %s emptySem failed(%s)", pSched->label, strerror(errno)); if (msg.fp) (*(msg.fp))(&msg); @@ -160,22 +160,23 @@ int taosScheduleTask(void *qhandle, SSchedMsg *pMsg) { while (tsem_wait(&pSched->emptySem) != 0) { if (errno != EINTR) { - pError("wait %s emptySem failed, reason:%s", pSched->label, strerror(errno)); + pError("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); break; } pTrace("wait %s emptySem was interrupted", pSched->label); } if (pthread_mutex_lock(&pSched->queueMutex) != 0) - pError("lock %s queueMutex failed, reason:%s", pSched->label, strerror(errno)); + pError("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); pSched->queue[pSched->emptySlot] = *pMsg; pSched->emptySlot = (pSched->emptySlot + 1) % pSched->queueSize; if (pthread_mutex_unlock(&pSched->queueMutex) != 0) - pError("unlock %s queueMutex failed, reason:%s", pSched->label, strerror(errno)); + pError("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); - if (tsem_post(&pSched->fullSem) != 0) pError("post %s fullSem failed, reason:%s", pSched->label, strerror(errno)); + if (tsem_post(&pSched->fullSem) != 0) + pError("post %s fullSem failed(%s)", pSched->label, strerror(errno)); return 0; } @@ -185,10 +186,12 @@ void taosCleanUpScheduler(void *param) { if (pSched == NULL) return; for (int i = 0; i < pSched->numOfThreads; ++i) { - pthread_cancel(pSched->qthread[i]); + if (pSched->qthread[i]) + pthread_cancel(pSched->qthread[i]); } for (int i = 0; i < pSched->numOfThreads; ++i) { - pthread_join(pSched->qthread[i], NULL); + if (pSched->qthread[i]) + pthread_join(pSched->qthread[i], NULL); } tsem_destroy(&pSched->emptySem); @@ -199,8 +202,8 @@ void taosCleanUpScheduler(void *param) { taosTmrStopA(&pSched->pTimer); } - free(pSched->queue); - free(pSched->qthread); + if (pSched->queue) free(pSched->queue); + if (pSched->qthread) free(pSched->qthread); free(pSched); // fix memory leak } From bd63cf2b4cc63c6ef386936722b4df7b5668ebe9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 12 Apr 2020 13:44:49 +0800 Subject: [PATCH 61/78] add general/user/basic1.sim --- tests/script/basicSuite.sim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index 440ec5592b..c282480bbe 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -10,4 +10,6 @@ run general/db/basic3.sim run general/db/basic4.sim run general/db/basic5.sim +run general/user/basic1.sim + ################################## From 50d7e6876d3a7e281be267dc6326f6bd0e84b32e Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 12 Apr 2020 13:49:34 +0800 Subject: [PATCH 62/78] remove possible memory leaks --- src/rpc/src/rpcServer.c | 86 ++++++++++++++++++++++------------------- src/rpc/src/rpcUdp.c | 10 ++--- 2 files changed, 50 insertions(+), 46 deletions(-) diff --git a/src/rpc/src/rpcServer.c b/src/rpc/src/rpcServer.c index 538b3059e3..6fe5385bba 100644 --- a/src/rpc/src/rpcServer.c +++ b/src/rpc/src/rpcServer.c @@ -65,79 +65,85 @@ static void taosProcessTcpData(void *param); static void taosAcceptTcpConnection(void *arg); void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { - int i; SServerObj *pServerObj; - pthread_attr_t thattr; SThreadObj *pThreadObj; - pServerObj = (SServerObj *)malloc(sizeof(SServerObj)); + pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1); strcpy(pServerObj->ip, ip); pServerObj->port = port; strcpy(pServerObj->label, label); pServerObj->numOfThreads = numOfThreads; - pServerObj->pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj) * (size_t)numOfThreads); + pServerObj->pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), numOfThreads); if (pServerObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); + free(pServerObj); return NULL; } - memset(pServerObj->pThreadObj, 0, sizeof(SThreadObj) * (size_t)numOfThreads); - - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + int code = 0; pThreadObj = pServerObj->pThreadObj; - for (i = 0; i < numOfThreads; ++i) { + for (int i = 0; i < numOfThreads; ++i) { pThreadObj->processData = fp; strcpy(pThreadObj->label, label); pThreadObj->shandle = shandle; - if (pthread_mutex_init(&(pThreadObj->threadMutex), NULL) < 0) { - tError("%s failed to init TCP process data mutex, reason:%s", label, strerror(errno)); - return NULL; + code = pthread_mutex_init(&(pThreadObj->threadMutex), NULL); + if (code < 0) { + tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); + break;; } - if (pthread_cond_init(&(pThreadObj->fdReady), NULL) != 0) { - tError("%s init TCP condition variable failed, reason:%s\n", label, strerror(errno)); - return NULL; + code = pthread_cond_init(&(pThreadObj->fdReady), NULL); + if (code != 0) { + tError("%s init TCP condition variable failed(%s)", label, strerror(errno)); + break; } pThreadObj->pollFd = epoll_create(10); // size does not matter if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP epoll", label); - return NULL; + code = -1; + break; } - if (pthread_create(&(pThreadObj->thread), &thattr, (void *)taosProcessTcpData, (void *)(pThreadObj)) != 0) { - tError("%s failed to create TCP process data thread, reason:%s", label, strerror(errno)); - return NULL; + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + code = pthread_create(&(pThreadObj->thread), &thattr, (void *)taosProcessTcpData, (void *)(pThreadObj)); + pthread_attr_destroy(&thattr); + if (code != 0) { + tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); + break; } pThreadObj->threadId = i; pThreadObj++; } - if (pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj)) != 0) { - tError("%s failed to create TCP accept thread, reason:%s", label, strerror(errno)); - return NULL; + if (code == 0) { + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + code = pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj)); + pthread_attr_destroy(&thattr); + if (code != 0) { + tError("%s failed to create TCP accept thread(%s)", label, strerror(errno)); + } } - /* - if ( pthread_create(&(pServerObj->thread), &thattr, - (void*)taosAcceptUDConnection, (void *)(pServerObj)) != 0 ) { - tError("%s failed to create UD accept thread, reason:%s", label, - strerror(errno)); - return NULL; - } - */ - pthread_attr_destroy(&thattr); - tTrace("%s TCP server is initialized, ip:%s port:%hu numOfThreads:%d", label, ip, port, numOfThreads); + if (code != 0) { + free(pServerObj->pThreadObj); + free(pServerObj); + pServerObj = NULL; + } else { + tTrace("%s TCP server is initialized, ip:%s port:%hu numOfThreads:%d", label, ip, port, numOfThreads); + } return (void *)pServerObj; } void taosCleanUpTcpServer(void *handle) { - int i; SThreadObj *pThreadObj; SServerObj *pServerObj = (SServerObj *)handle; @@ -146,7 +152,7 @@ void taosCleanUpTcpServer(void *handle) { pthread_cancel(pServerObj->thread); pthread_join(pServerObj->thread, NULL); - for (i = 0; i < pServerObj->numOfThreads; ++i) { + for (int i = 0; i < pServerObj->numOfThreads; ++i) { pThreadObj = pServerObj->pThreadObj + i; while (pThreadObj->pHead) { @@ -161,9 +167,9 @@ void taosCleanUpTcpServer(void *handle) { pthread_mutex_destroy(&(pThreadObj->threadMutex)); } - tfree(pServerObj->pThreadObj); tTrace("TCP:%s, TCP server is cleaned up", pServerObj->label); + tfree(pServerObj->pThreadObj); tfree(pServerObj); } @@ -278,10 +284,10 @@ static void taosAcceptTcpConnection(void *arg) { sockFd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port); if (sockFd < 0) { - tError("%s failed to open TCP socket, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); + tError("%s failed to open TCP socket, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); return; } else { - tTrace("%s TCP server is ready, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); + tTrace("%s TCP server is ready, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); } while (1) { @@ -289,11 +295,11 @@ static void taosAcceptTcpConnection(void *arg) { connFd = accept(sockFd, (struct sockaddr *)&clientAddr, &addrlen); if (connFd < 0) { - tError("%s TCP accept failure, errno:%d, reason:%s", pServerObj->label, errno, strerror(errno)); + tError("%s TCP accept failure(%s)", pServerObj->label, errno, strerror(errno)); continue; } - tTrace("%s TCP connection from ip:%s port:%hu", pServerObj->label, inet_ntoa(clientAddr.sin_addr), + tTrace("%s TCP connection from ip:%s:%hu", pServerObj->label, inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); taosKeepTcpAlive(connFd); @@ -318,7 +324,7 @@ static void taosAcceptTcpConnection(void *arg) { event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; event.data.ptr = pFdObj; if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { - tError("%s failed to add TCP FD for epoll, error:%s", pServerObj->label, strerror(errno)); + tError("%s failed to add TCP FD for epoll(%s)", pServerObj->label, strerror(errno)); tfree(pFdObj); close(connFd); continue; diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 785288f5b9..e666187cf1 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -78,7 +78,6 @@ static SUdpBuf *taosCreateUdpBuf(SUdpConn *pConn, uint32_t ip, uint16_t port); static void taosProcessUdpBufTimer(void *param, void *tmrId); void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, void *fp, void *shandle) { - pthread_attr_t thAttr; SUdpConn * pConn; SUdpConnSet * pSet; @@ -106,9 +105,6 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v } } - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - uint16_t ownPort; for (int i = 0; i < threads; ++i) { pConn = pSet->udpConn + i; @@ -146,19 +142,21 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v pConn->tmrCtrl = pSet->tmrCtrl; } + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn); + pthread_attr_destroy(&thAttr); if (code != 0) { tError("%s failed to create thread to process UDP data, reason:%s", label, strerror(errno)); taosCloseSocket(pConn->fd); taosCleanUpUdpConnection(pSet); - pthread_attr_destroy(&thAttr); return NULL; } ++pSet->threads; } - pthread_attr_destroy(&thAttr); tTrace("%s UDP connection is initialized, ip:%s port:%hu threads:%d", label, ip, port, threads); return pSet; From a5dabf7e9919851c861d3d9fa182de109733a884 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 12 Apr 2020 17:05:06 +0800 Subject: [PATCH 63/78] remove the potential race condition on TCP FD --- src/rpc/src/rpcServer.c | 47 ++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/src/rpc/src/rpcServer.c b/src/rpc/src/rpcServer.c index 6fe5385bba..7430157bcf 100644 --- a/src/rpc/src/rpcServer.c +++ b/src/rpc/src/rpcServer.c @@ -145,7 +145,7 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void taosCleanUpTcpServer(void *handle) { SThreadObj *pThreadObj; - SServerObj *pServerObj = (SServerObj *)handle; + SServerObj *pServerObj = handle; if (pServerObj == NULL) return; @@ -174,15 +174,15 @@ void taosCleanUpTcpServer(void *handle) { } void taosCloseTcpServerConnection(void *chandle) { - SFdObj *pFdObj = (SFdObj *)chandle; - + SFdObj *pFdObj = chandle; if (pFdObj == NULL) return; + pFdObj->thandle = NULL; taosCleanUpFdObj(pFdObj); } int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { - SFdObj *pFdObj = (SFdObj *)chandle; + SFdObj *pFdObj = chandle; if (chandle == NULL) return -1; @@ -354,14 +354,25 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { if (pFdObj == NULL) return; if (pFdObj->signature != pFdObj) return; + pFdObj->signature = NULL; pThreadObj = pFdObj->pThreadObj; - if (pThreadObj == NULL) { - tError("FdObj double clean up!!!"); - return; - } - epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); + // notify the upper layer, so it will clean the associated context + if (pFdObj->thandle) { + SRecvInfo recvInfo; + recvInfo.msg = NULL; + recvInfo.msgLen = 0; + recvInfo.ip = 0; + recvInfo.port = 0; + recvInfo.shandle = pThreadObj->shandle; + recvInfo.thandle = pFdObj->thandle;; + recvInfo.chandle = NULL; + recvInfo.connType = RPC_CONN_TCP; + (*(pThreadObj->processData))(&recvInfo); + } + close(pFdObj->fd); + epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); pthread_mutex_lock(&pThreadObj->threadMutex); @@ -384,24 +395,8 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->threadMutex); - // notify the upper layer, so it will clean the associated context - SRecvInfo recvInfo; - recvInfo.msg = NULL; - recvInfo.msgLen = 0; - recvInfo.ip = 0; - recvInfo.port = 0; - recvInfo.shandle = pThreadObj->shandle; - recvInfo.thandle = pFdObj->thandle;; - recvInfo.chandle = NULL; - recvInfo.connType = RPC_CONN_TCP; - - if (pFdObj->thandle) (*(pThreadObj->processData))(&recvInfo); - tTrace("%s TCP thread:%d, FD:%p is cleaned up, numOfFds:%d", pThreadObj->label, pThreadObj->threadId, - pFdObj, pThreadObj->numOfFds); - - memset(pFdObj, 0, sizeof(SFdObj)); - + pFdObj, pThreadObj->numOfFds); tfree(pFdObj); } From 8c22eed1bbfe2f0317c63be88ef0420087ba422e Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 12 Apr 2020 18:10:06 +0800 Subject: [PATCH 64/78] remove a potential race condition --- src/rpc/src/rpcClient.c | 26 ++++++++++++-------------- src/rpc/test/rclient.c | 2 ++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/rpc/src/rpcClient.c b/src/rpc/src/rpcClient.c index 264449bbb0..a3105ef516 100644 --- a/src/rpc/src/rpcClient.c +++ b/src/rpc/src/rpcClient.c @@ -188,10 +188,19 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { if (pFdObj == NULL) return; if (pFdObj->signature != pFdObj) return; + pFdObj->signature = NULL; pTcp = pFdObj->pTcp; - if (pTcp == NULL) { - tError("double free TcpFdObj!!!!"); - return; + + if (pFdObj->thandle) { + recvInfo.msg = NULL; + recvInfo.msgLen = 0; + recvInfo.ip = 0; + recvInfo.port = 0; + recvInfo.shandle = pTcp->shandle; + recvInfo.thandle = pFdObj->thandle;; + recvInfo.chandle = NULL; + recvInfo.connType = RPC_CONN_TCP; + (*(pTcp->processData))(&recvInfo); } epoll_ctl(pTcp->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); @@ -216,19 +225,8 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { pthread_mutex_unlock(&pTcp->mutex); - recvInfo.msg = NULL; - recvInfo.msgLen = 0; - recvInfo.ip = 0; - recvInfo.port = 0; - recvInfo.shandle = pTcp->shandle; - recvInfo.thandle = pFdObj->thandle;; - recvInfo.chandle = NULL; - recvInfo.connType = RPC_CONN_TCP; - - if (pFdObj->thandle) (*(pTcp->processData))(&recvInfo); tTrace("%s TCP is cleaned up, FD:%p numOfFds:%d", pTcp->label, pFdObj, pTcp->numOfFds); - memset(pFdObj, 0, sizeof(STcpFd)); tfree(pFdObj); } diff --git a/src/rpc/test/rclient.c b/src/rpc/test/rclient.c index f000ab91a2..732d7eb81c 100644 --- a/src/rpc/test/rclient.c +++ b/src/rpc/test/rclient.c @@ -204,6 +204,8 @@ int main(int argc, char *argv[]) { tPrint("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs*appThreads); tPrint("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize); + getchar(); + taosCloseLogger(); return 0; From f5a1ac4b48a3b45fcff16045a9285cab559b2009 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sun, 12 Apr 2020 21:46:34 +0800 Subject: [PATCH 65/78] [td-98]fix bugs in descending order query for tables --- src/client/src/tscFunctionImpl.c | 115 ------ src/client/src/tscSQLParser.c | 1 + src/mnode/src/mgmtTable.c | 9 +- src/query/inc/queryExecutor.h | 4 +- src/query/inc/tsqlfunction.h | 16 - src/query/src/queryExecutor.c | 29 +- src/util/inc/tcompare.h | 24 ++ src/util/src/tcompare.c | 139 ++++++++ src/vnode/tsdb/src/tsdbRead.c | 590 ++++++++++++------------------- 9 files changed, 426 insertions(+), 501 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 0fa9e01c2e..a5a0bf4a78 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -3578,121 +3578,6 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } -/* - * Compare two strings - * TSDB_MATCH: Match - * TSDB_NOMATCH: No match - * TSDB_NOWILDCARDMATCH: No match in spite of having * or % wildcards. - * Like matching rules: - * '%': Matches zero or more characters - * '_': Matches one character - * - */ -int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) { - char c, c1; - - int32_t i = 0; - int32_t j = 0; - - while ((c = patterStr[i++]) != 0) { - if (c == pInfo->matchAll) { /* Match "*" */ - - while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) { - if (c == pInfo->matchOne && (j > size || str[j++] == 0)) { - // empty string, return not match - return TSDB_PATTERN_NOWILDCARDMATCH; - } - } - - if (c == 0) { - return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */ - } - - char next[3] = {toupper(c), tolower(c), 0}; - while (1) { - size_t n = strcspn(str, next); - str += n; - - if (str[0] == 0 || (n >= size - 1)) { - break; - } - - int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo); - if (ret != TSDB_PATTERN_NOMATCH) { - return ret; - } - } - return TSDB_PATTERN_NOWILDCARDMATCH; - } - - c1 = str[j++]; - - if (j <= size) { - if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) { - continue; - } - } - - return TSDB_PATTERN_NOMATCH; - } - - return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; -} - -int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) { - wchar_t c, c1; - wchar_t matchOne = L'_'; // "_" - wchar_t matchAll = L'%'; // "%" - - int32_t i = 0; - int32_t j = 0; - - while ((c = patterStr[i++]) != 0) { - if (c == matchAll) { /* Match "%" */ - - while ((c = patterStr[i++]) == matchAll || c == matchOne) { - if (c == matchOne && (j > size || str[j++] == 0)) { - return TSDB_PATTERN_NOWILDCARDMATCH; - } - } - if (c == 0) { - return TSDB_PATTERN_MATCH; - } - - wchar_t accept[3] = {towupper(c), towlower(c), 0}; - while (1) { - size_t n = wcsspn(str, accept); - - str += n; - if (str[0] == 0 || (n >= size - 1)) { - break; - } - - str++; - - int32_t ret = WCSPatternMatch(&patterStr[i], str, wcslen(str), pInfo); - if (ret != TSDB_PATTERN_NOMATCH) { - return ret; - } - } - - return TSDB_PATTERN_NOWILDCARDMATCH; - } - - c1 = str[j++]; - - if (j <= size) { - if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) { - continue; - } - } - - return TSDB_PATTERN_NOMATCH; - } - - return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; -} - static void getStatics_i8(int64_t *primaryKey, int32_t type, int8_t *data, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { *min = INT64_MAX; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 630e439397..5657b6b47d 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -30,6 +30,7 @@ #include "ttokendef.h" #include "name.h" +#include "tcompare.h" #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index b536bb5ac9..e6d597f5f5 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -15,19 +15,13 @@ #define _DEFAULT_SOURCE #include "os.h" + #include "taosmsg.h" -#include "tscompression.h" -#include "tskiplist.h" #include "ttime.h" #include "tutil.h" -#include "qast.h" -#include "qextbuffer.h" #include "taoserror.h" #include "taosmsg.h" #include "tscompression.h" -#include "tskiplist.h" -#include "tsqlfunction.h" -#include "ttime.h" #include "name.h" #include "taccount.h" #include "mgmtDClient.h" @@ -42,6 +36,7 @@ #include "mgmtTable.h" #include "mgmtUser.h" #include "mgmtVgroup.h" +#include "tcompare.h" void * tsChildTableSdb; void * tsSuperTableSdb; diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 5aab3abf40..3932650e1f 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -160,7 +160,7 @@ typedef struct SQueryRuntimeEnv { SQueryCostSummary summary; bool stableQuery; // super table query or not void* pQueryHandle; - + void* pSubQueryHandle; // another thread for SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file } SQueryRuntimeEnv; @@ -172,6 +172,8 @@ typedef struct SQInfo { int32_t code; // error code to returned to client sem_t dataReady; SArray* pTableIdList; // table id list + void* tsdb; + SQueryRuntimeEnv runtimeEnv; int32_t subgroupIdx; int32_t offset; /* offset in group result set of subgroup */ diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 7514f1e44c..90ae7df36a 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -79,17 +79,10 @@ extern "C" { #define TSDB_BASE_FUNC_SO TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF #define TSDB_BASE_FUNC_MO TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF -#define TSDB_PATTERN_MATCH 0 -#define TSDB_PATTERN_NOMATCH 1 -#define TSDB_PATTERN_NOWILDCARDMATCH 2 -#define TSDB_PATTERN_STRING_MAX_LEN 20 #define TSDB_FUNCTIONS_NAME_MAX_LENGTH 16 #define TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE 50 -#define PATTERN_COMPARE_INFO_INITIALIZER \ - { '%', '_' } - #define DATA_SET_FLAG ',' // to denote the output area has data, not null value #define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG) @@ -222,20 +215,11 @@ typedef struct SQLAggFuncElem { int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId); } SQLAggFuncElem; -typedef struct SPatternCompareInfo { - char matchAll; // symbol for match all wildcard, default: '%' - char matchOne; // symbol for match one wildcard, default: '_' -} SPatternCompareInfo; - #define GET_RES_INFO(ctx) ((ctx)->resultInfo) int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, int16_t *len, int16_t *interResBytes, int16_t extLength, bool isSuperTable); -int patternMatch(const char *zPattern, const char *zString, size_t size, const SPatternCompareInfo *pInfo); - -int WCSPatternMatch(const wchar_t *zPattern, const wchar_t *zString, size_t size, const SPatternCompareInfo *pInfo); - #define IS_STREAM_QUERY_VALID(x) (((x)&TSDB_FUNCSTATE_STREAM) != 0) #define IS_MULTIOUTPUT(x) (((x)&TSDB_FUNCSTATE_MO) != 0) #define IS_SINGLEOUTPUT(x) (((x)&TSDB_FUNCSTATE_SO) != 0) diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 6e540f6ae1..0f74cf1beb 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -2565,7 +2565,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order); - tsdb_query_handle_t pQueryHandle = pRuntimeEnv->pQueryHandle; + tsdb_query_handle_t pQueryHandle = pRuntimeEnv->scanFlag == MASTER_SCAN? pRuntimeEnv->pQueryHandle:pRuntimeEnv->pSubQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { @@ -3520,8 +3520,9 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { setQueryStatus(pQuery, QUERY_NOT_COMPLETED); // store the start query position - void *pos = tsdbDataBlockTell(pRuntimeEnv->pQueryHandle); - +// void *pos = tsdbDataBlockTell(pRuntimeEnv->pQueryHandle); + SQInfo* pQInfo = (SQInfo*) GET_QINFO_ADDR(pRuntimeEnv); + int64_t skey = pQuery->lastKey; int32_t status = pQuery->status; int32_t activeSlot = pRuntimeEnv->windowResInfo.curIndex; @@ -3543,15 +3544,29 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { } // set the correct start position, and load the corresponding block in buffer for next round scan all data blocks. - /*int32_t ret =*/ tsdbDataBlockSeek(pRuntimeEnv->pQueryHandle, pos); - +// /*int32_t ret =*/ tsdbDataBlockSeek(pRuntimeEnv->pQueryHandle, pos); + + STsdbQueryCond cond = { + .twindow = {pQuery->window.skey, pQuery->lastKey}, + .order = pQuery->order.order, + .colList = pQuery->colList, + }; + + SArray *cols = taosArrayInit(pQuery->numOfCols, sizeof(pQuery->colList[0])); + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + taosArrayPush(cols, &pQuery->colList[i]); + } + + pRuntimeEnv->pSubQueryHandle = tsdbQueryByTableId(pQInfo->tsdb, &cond, pQInfo->pTableIdList, cols); + taosArrayDestroy(cols); + status = pQuery->status; pRuntimeEnv->windowResInfo.curIndex = activeSlot; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); pRuntimeEnv->scanFlag = REPEAT_SCAN; - /* check if query is killed or not */ + // check if query is killed or not if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { return; } @@ -4179,6 +4194,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) pRuntimeEnv->pQueryHandle = tsdbQueryByTableId(tsdb, &cond, pQInfo->pTableIdList, cols); taosArrayDestroy(cols); + pQInfo->tsdb = tsdb; pRuntimeEnv->pQuery = pQuery; pRuntimeEnv->pTSBuf = param; @@ -4972,7 +4988,6 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; while (1) { -// initCtxOutputBuf(pRuntimeEnv); scanAllDataBlocks(pRuntimeEnv); if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { diff --git a/src/util/inc/tcompare.h b/src/util/inc/tcompare.h index 3ac0d3d170..b8778b80c8 100644 --- a/src/util/inc/tcompare.h +++ b/src/util/inc/tcompare.h @@ -16,8 +16,24 @@ #ifndef TDENGINE_TCOMPARE_H #define TDENGINE_TCOMPARE_H +#ifdef __cplusplus +extern "C" { +#endif + #include "os.h" +#define TSDB_PATTERN_MATCH 0 +#define TSDB_PATTERN_NOMATCH 1 +#define TSDB_PATTERN_NOWILDCARDMATCH 2 +#define TSDB_PATTERN_STRING_MAX_LEN 20 + +#define PATTERN_COMPARE_INFO_INITIALIZER { '%', '_' } + +typedef struct SPatternCompareInfo { + char matchAll; // symbol for match all wildcard, default: '%' + char matchOne; // symbol for match one wildcard, default: '_' +} SPatternCompareInfo; + int32_t compareInt32Val(const void *pLeft, const void *pRight); int32_t compareInt64Val(const void *pLeft, const void *pRight); @@ -36,8 +52,16 @@ int32_t compareStrVal(const void *pLeft, const void *pRight); int32_t compareWStrVal(const void *pLeft, const void *pRight); +int patternMatch(const char *zPattern, const char *zString, size_t size, const SPatternCompareInfo *pInfo); + +int WCSPatternMatch(const wchar_t *zPattern, const wchar_t *zString, size_t size, const SPatternCompareInfo *pInfo); + __compar_fn_t getKeyComparFunc(int32_t keyType); __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType); +#ifdef __cplusplus +} +#endif + #endif // TDENGINE_TCOMPARE_H diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 401db9279a..bff10c3022 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -1,6 +1,8 @@ #include "taosdef.h" #include "tcompare.h" +#include "tutil.h" + int32_t compareInt32Val(const void *pLeft, const void *pRight) { int32_t ret = GET_INT32_VAL(pLeft) - GET_INT32_VAL(pRight); if (ret == 0) { @@ -102,6 +104,143 @@ int32_t compareWStrVal(const void *pLeft, const void *pRight) { return 0; } +/* + * Compare two strings + * TSDB_MATCH: Match + * TSDB_NOMATCH: No match + * TSDB_NOWILDCARDMATCH: No match in spite of having * or % wildcards. + * Like matching rules: + * '%': Matches zero or more characters + * '_': Matches one character + * + */ +int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) { + char c, c1; + + int32_t i = 0; + int32_t j = 0; + + while ((c = patterStr[i++]) != 0) { + if (c == pInfo->matchAll) { /* Match "*" */ + + while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) { + if (c == pInfo->matchOne && (j > size || str[j++] == 0)) { + // empty string, return not match + return TSDB_PATTERN_NOWILDCARDMATCH; + } + } + + if (c == 0) { + return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */ + } + + char next[3] = {toupper(c), tolower(c), 0}; + while (1) { + size_t n = strcspn(str, next); + str += n; + + if (str[0] == 0 || (n >= size - 1)) { + break; + } + + int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo); + if (ret != TSDB_PATTERN_NOMATCH) { + return ret; + } + } + return TSDB_PATTERN_NOWILDCARDMATCH; + } + + c1 = str[j++]; + + if (j <= size) { + if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) { + continue; + } + } + + return TSDB_PATTERN_NOMATCH; + } + + return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; +} + +int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) { + wchar_t c, c1; + wchar_t matchOne = L'_'; // "_" + wchar_t matchAll = L'%'; // "%" + + int32_t i = 0; + int32_t j = 0; + + while ((c = patterStr[i++]) != 0) { + if (c == matchAll) { /* Match "%" */ + + while ((c = patterStr[i++]) == matchAll || c == matchOne) { + if (c == matchOne && (j > size || str[j++] == 0)) { + return TSDB_PATTERN_NOWILDCARDMATCH; + } + } + if (c == 0) { + return TSDB_PATTERN_MATCH; + } + + wchar_t accept[3] = {towupper(c), towlower(c), 0}; + while (1) { + size_t n = wcsspn(str, accept); + + str += n; + if (str[0] == 0 || (n >= size - 1)) { + break; + } + + str++; + + int32_t ret = WCSPatternMatch(&patterStr[i], str, wcslen(str), pInfo); + if (ret != TSDB_PATTERN_NOMATCH) { + return ret; + } + } + + return TSDB_PATTERN_NOWILDCARDMATCH; + } + + c1 = str[j++]; + + if (j <= size) { + if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) { + continue; + } + } + + return TSDB_PATTERN_NOMATCH; + } + + return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; +} + +static UNUSED_FUNC int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + const char* pattern = pRight; + const char* str = pLeft; + + int32_t ret = patternMatch(pattern, str, strlen(str), &pInfo); + + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; +} + +static UNUSED_FUNC int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + const wchar_t* pattern = pRight; + const wchar_t* str = pLeft; + + int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo); + + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; +} + __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType) { __compar_fn_t comparFn = NULL; diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 2fce73e547..966debda8d 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -87,6 +87,13 @@ typedef struct STableBlockInfo { int32_t groupIdx; /* number of group is less than the total number of tables */ } STableBlockInfo; +typedef struct SBlockOrderSupporter { + int32_t numOfTables; + STableBlockInfo** pDataBlockInfo; + int32_t* blockIndexArray; + int32_t* numOfBlocksPerMeter; +} SBlockOrderSupporter; + typedef struct STsdbQueryHandle { STsdbRepo* pTsdb; SQueryFilePos cur; // current position @@ -222,7 +229,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { // todo dynamic get the daysperfile static int32_t getFileIdFromKey(TSKEY key) { - int64_t fid = (int64_t)(key / 10); // set the starting fileId + int64_t fid = (int64_t)(key / (10 * tsMsPerDay[0])); // set the starting fileId if (fid > INT32_MAX) { fid = INT32_MAX; } @@ -230,7 +237,32 @@ static int32_t getFileIdFromKey(TSKEY key) { return fid; } -static int32_t binarySearchForBlockImpl(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order); +static int32_t binarySearchForBlockImpl(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { + int32_t firstSlot = 0; + int32_t lastSlot = numOfBlocks - 1; + + int32_t midSlot = firstSlot; + + while (1) { + numOfBlocks = lastSlot - firstSlot + 1; + midSlot = (firstSlot + (numOfBlocks >> 1)); + + if (numOfBlocks == 1) break; + + if (skey > pBlock[midSlot].keyLast) { + if (numOfBlocks == 2) break; + if ((order == TSDB_ORDER_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break; + firstSlot = midSlot + 1; + } else if (skey < pBlock[midSlot].keyFirst) { + if ((order == TSDB_ORDER_ASC) && (skey > pBlock[midSlot - 1].keyLast)) break; + lastSlot = midSlot - 1; + } else { + break; // got the slot + } + } + + return midSlot; +} static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks, int32_t type) { // todo check open file failed @@ -299,33 +331,6 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo return TSDB_CODE_SUCCESS; } -int32_t binarySearchForBlockImpl(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { - int32_t firstSlot = 0; - int32_t lastSlot = numOfBlocks - 1; - - int32_t midSlot = firstSlot; - - while (1) { - numOfBlocks = lastSlot - firstSlot + 1; - midSlot = (firstSlot + (numOfBlocks >> 1)); - - if (numOfBlocks == 1) break; - - if (skey > pBlock[midSlot].keyLast) { - if (numOfBlocks == 2) break; - if ((order == TSDB_ORDER_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break; - firstSlot = midSlot + 1; - } else if (skey < pBlock[midSlot].keyFirst) { - if ((order == TSDB_ORDER_ASC) && (skey > pBlock[midSlot - 1].keyLast)) break; - lastSlot = midSlot - 1; - } else { - break; // got the slot - } - } - - return midSlot; -} - static SDataBlockInfo getTrueDataBlockInfo(STableCheckInfo* pCheckInfo, SCompBlock* pBlock) { SDataBlockInfo info = { .window = {.skey = pBlock->keyFirst, .ekey = pBlock->keyLast}, @@ -338,7 +343,34 @@ static SDataBlockInfo getTrueDataBlockInfo(STableCheckInfo* pCheckInfo, SCompBlo return info; } -SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS); +static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) { + size_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); + assert(numOfCols <= TSDB_MAX_COLUMNS); + + SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); + taosArrayPush(pIdList, &pCol->info.colId); + } + + return pIdList; +} + +static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS) { + SArray* pLocalIdList = getColumnIdList(pQueryHandle); + + // check if the primary time stamp column needs to load + int16_t colId = *(int16_t*)taosArrayGet(pLocalIdList, 0); + + // the primary timestamp column does not be included in the the specified load column list, add it + if (loadTS && colId != 0) { + int16_t columnId = 0; + taosArrayInsert(pLocalIdList, 0, &columnId); + } + + return pLocalIdList; +} + static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock, SArray* sa); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); @@ -388,7 +420,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { // query ended in current block - if (pQueryHandle->window.ekey < pBlock->keyLast) { + if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { return false; } @@ -430,62 +462,62 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock return pQueryHandle->realNumOfRows > 0; } -bool moveToNextBlock(STsdbQueryHandle* pQueryHandle, int32_t step) { - SQueryFilePos* cur = &pQueryHandle->cur; +//bool moveToNextBlock(STsdbQueryHandle* pQueryHandle, int32_t step) { +// SQueryFilePos* cur = &pQueryHandle->cur; +// +// if (pQueryHandle->cur.fid >= 0) { +// /* +// * 1. ascending order. The last data block of data file +// * 2. descending order. The first block of file +// */ +// STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); +// int32_t tid = pCheckInfo->tableId.tid; +// +// if ((step == QUERY_ASC_FORWARD_STEP && +// (pQueryHandle->cur.slot == pQueryHandle->compIndex[tid].numOfSuperBlocks - 1)) || +// (step == QUERY_DESC_FORWARD_STEP && (pQueryHandle->cur.slot == 0))) { +// // temporarily keep the position value, in case of no data qualified when move forwards(backwards) +// // SQueryFilePos save = pQueryHandle->cur; +// pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); +// +// int32_t fid = -1; +// int32_t numOfBlocks = 0; +// +// if (pQueryHandle->pFileGroup != NULL) { +// if ((fid = getFileCompInfo(pQueryHandle, &numOfBlocks, 1)) < 0) { +// } else { +// cur->slot = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->numOfBlocks - 1; +// cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->pBlock[cur->slot].numOfPoints - 1; +// +// SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; +// cur->fid = pQueryHandle->pFileGroup->fileId; +// assert(cur->pos >= 0 && cur->fid >= 0 && cur->slot >= 0); +// +// if (pBlock->keyFirst > pQueryHandle->window.ekey) { // done +// return false; +// } +// +// return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); +// } +// } else { // check data in cache +// pQueryHandle->cur.fid = -1; +// return hasMoreDataInCache(pQueryHandle); +// } +// } else { // next block in the same file +// cur->slot += step; +// +// SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; +// cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pBlock->numOfPoints - 1; +// return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); +// } +// } else { // data in cache +// return hasMoreDataInCache(pQueryHandle); +// } +// +// return false; +//} - if (pQueryHandle->cur.fid >= 0) { - /* - * 1. ascending order. The last data block of data file - * 2. descending order. The first block of file - */ - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - int32_t tid = pCheckInfo->tableId.tid; - - if ((step == QUERY_ASC_FORWARD_STEP && - (pQueryHandle->cur.slot == pQueryHandle->compIndex[tid].numOfSuperBlocks - 1)) || - (step == QUERY_DESC_FORWARD_STEP && (pQueryHandle->cur.slot == 0))) { - // temporarily keep the position value, in case of no data qualified when move forwards(backwards) - // SQueryFilePos save = pQueryHandle->cur; - pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); - - int32_t fid = -1; - int32_t numOfBlocks = 0; - - if (pQueryHandle->pFileGroup != NULL) { - if ((fid = getFileCompInfo(pQueryHandle, &numOfBlocks, 1)) < 0) { - } else { - cur->slot = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->numOfBlocks - 1; - cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->pBlock[cur->slot].numOfPoints - 1; - - SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; - cur->fid = pQueryHandle->pFileGroup->fileId; - assert(cur->pos >= 0 && cur->fid >= 0 && cur->slot >= 0); - - if (pBlock->keyFirst > pQueryHandle->window.ekey) { // done - return false; - } - - return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); - } - } else { // check data in cache - pQueryHandle->cur.fid = -1; - return hasMoreDataInCache(pQueryHandle); - } - } else { // next block in the same file - cur->slot += step; - - SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; - cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pBlock->numOfPoints - 1; - return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); - } - } else { // data in cache - return hasMoreDataInCache(pQueryHandle); - } - - return false; -} - -int vnodeBinarySearchKey(char* pValue, int num, TSKEY key, int order) { +static int vnodeBinarySearchKey(char* pValue, int num, TSKEY key, int order) { int firstPos, lastPos, midPos = -1; int numOfPoints; TSKEY* keyList; @@ -572,7 +604,7 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf pQueryHandle->realNumOfRows = 0; return; } else { - pQueryHandle->realNumOfRows = endPos - cur->pos; + pQueryHandle->realNumOfRows = endPos - cur->pos + 1; } pCheckInfo->lastKey = ((int64_t*)(pCols->cols[0].pData))[endPos] + 1; @@ -581,10 +613,8 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf pQueryHandle->realNumOfRows = 0; return; } else { - pQueryHandle->realNumOfRows = cur->pos - endPos; + pQueryHandle->realNumOfRows = cur->pos - endPos + 1; } - - assert(0); } } @@ -614,34 +644,6 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf cur->pos = endPos; } -static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) { - size_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); - assert(numOfCols <= TSDB_MAX_COLUMNS); - - SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); - taosArrayPush(pIdList, &pCol->info.colId); - } - - return pIdList; -} - -SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS) { - SArray* pLocalIdList = getColumnIdList(pQueryHandle); - - // check if the primary time stamp column needs to load - int16_t colId = *(int16_t*)taosArrayGet(pLocalIdList, 0); - - // the primary timestamp column does not be included in the the specified load column list, add it - if (loadTS && colId != 0) { - int16_t columnId = 0; - taosArrayInsert(pLocalIdList, 0, &columnId); - } - - return pLocalIdList; -} - int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { int firstPos, lastPos, midPos = -1; int numOfPoints; @@ -702,79 +704,72 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { return midPos; } -static bool getQualifiedDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t type) { - STsdbFileH* pFileHandle = tsdbGetFile(pQueryHandle->pTsdb); - int32_t fid = getFileIdFromKey(pCheckInfo->lastKey); +//static bool getQualifiedDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t type) { +// STsdbFileH* pFileHandle = tsdbGetFile(pQueryHandle->pTsdb); +// int32_t fid = getFileIdFromKey(pCheckInfo->lastKey); +// +// tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, TSDB_FGROUP_ITER_FORWARD); +// tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); +// pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); +// +// SQueryFilePos* cur = &pQueryHandle->cur; +// +// int32_t tid = pCheckInfo->tableId.tid; +// int32_t numOfBlocks = 0; +// +// while (pQueryHandle->pFileGroup != NULL) { +// if (getFileCompInfo(pQueryHandle, &numOfBlocks, 1) != TSDB_CODE_SUCCESS) { +// break; +// } +// +// assert(pCheckInfo->numOfBlocks >= 0); +// +// // no data block in current file, try next +// if (pCheckInfo->numOfBlocks > 0) { +// cur->fid = pQueryHandle->pFileGroup->fileId; +// break; +// } +// +// dTrace("%p no data block in file, fid:%d, tid:%d, try next, %p", pQueryHandle, pQueryHandle->pFileGroup->fileId, +// tid, pQueryHandle->qinfo); +// +// pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); +// } +// +// if (pCheckInfo->numOfBlocks == 0) { +// return false; +// } +// +// cur->slot = 0; // always start from the first slot +// SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; +// return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); +//} - tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, TSDB_FGROUP_ITER_FORWARD); - tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); - pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); +//static UNUSED_FUNC bool hasMoreDataForSingleTable(STsdbQueryHandle* pHandle) { +// assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); +// +// STsdbFileH* pFileHandle = tsdbGetFile(pHandle->pTsdb); +// STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); +// +// if (!pCheckInfo->checkFirstFileBlock) { +// pCheckInfo->checkFirstFileBlock = true; +// +// if (pFileHandle != NULL) { +// bool found = getQualifiedDataBlock(pHandle, pCheckInfo, 1); +// if (found) { +// return true; +// } +// } +// +// // no data in file, try cache +// pHandle->cur.fid = -1; +// return hasMoreDataInCache(pHandle); +// } else { // move to next data block in file or in cache +// return moveToNextBlock(pHandle, 1); +// } +//} - SQueryFilePos* cur = &pQueryHandle->cur; - - int32_t tid = pCheckInfo->tableId.tid; - int32_t numOfBlocks = 0; - - while (pQueryHandle->pFileGroup != NULL) { - if (getFileCompInfo(pQueryHandle, &numOfBlocks, 1) != TSDB_CODE_SUCCESS) { - break; - } - - assert(pCheckInfo->numOfBlocks >= 0); - - // no data block in current file, try next - if (pCheckInfo->numOfBlocks > 0) { - cur->fid = pQueryHandle->pFileGroup->fileId; - break; - } - - dTrace("%p no data block in file, fid:%d, tid:%d, try next, %p", pQueryHandle, pQueryHandle->pFileGroup->fileId, - tid, pQueryHandle->qinfo); - - pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); - } - - if (pCheckInfo->numOfBlocks == 0) { - return false; - } - - cur->slot = 0; // always start from the first slot - SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; - return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); -} - -static UNUSED_FUNC bool hasMoreDataForSingleTable(STsdbQueryHandle* pHandle) { - assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); - - STsdbFileH* pFileHandle = tsdbGetFile(pHandle->pTsdb); - STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - - if (!pCheckInfo->checkFirstFileBlock) { - pCheckInfo->checkFirstFileBlock = true; - - if (pFileHandle != NULL) { - bool found = getQualifiedDataBlock(pHandle, pCheckInfo, 1); - if (found) { - return true; - } - } - - // no data in file, try cache - pHandle->cur.fid = -1; - return hasMoreDataInCache(pHandle); - } else { // move to next data block in file or in cache - return moveToNextBlock(pHandle, 1); - } -} - -typedef struct SBlockOrderSupporter { - int32_t numOfTables; - STableBlockInfo** pDataBlockInfo; - int32_t* blockIndexArray; - int32_t* numOfBlocksPerMeter; -} SBlockOrderSupporter; - -void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t numOfTables) { +static void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t numOfTables) { tfree(pSupporter->numOfBlocksPerMeter); tfree(pSupporter->blockIndexArray); @@ -815,7 +810,7 @@ static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* return pLeftBlockInfoEx->pBlock.compBlock->offset > pRightBlockInfoEx->pBlock.compBlock->offset ? 1 : -1; } -int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { +static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { char* tmp = realloc(pQueryHandle->pDataBlockInfo, sizeof(STableBlockInfo) * numOfBlocks); if (tmp == NULL) { return TSDB_CODE_SERV_OUT_OF_MEMORY; @@ -912,6 +907,53 @@ int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks return TSDB_CODE_SUCCESS; } +// todo opt for only one table case +static bool getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle) { + pQueryHandle->numOfBlocks = 0; + SQueryFilePos* cur = &pQueryHandle->cur; + + int32_t numOfBlocks = 0; + int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + + while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { + int32_t type = QUERY_IS_ASC_QUERY(pQueryHandle->order)? QUERY_RANGE_GREATER_EQUAL:QUERY_RANGE_LESS_EQUAL; + if (getFileCompInfo(pQueryHandle, &numOfBlocks, type) != TSDB_CODE_SUCCESS) { + break; + } + + assert(numOfBlocks >= 0); + dTrace("%p %d blocks found in file for %d table(s), fid:%d", pQueryHandle, numOfBlocks, + numOfTables, pQueryHandle->pFileGroup->fileId); + + // todo return error code to query engine + if (createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks) != TSDB_CODE_SUCCESS) { + break; + } + + assert(numOfBlocks >= pQueryHandle->numOfBlocks); + if (pQueryHandle->numOfBlocks > 0) { + break; + } + } + + // no data in file anymore + if (pQueryHandle->numOfBlocks <= 0) { + assert(pQueryHandle->pFileGroup == NULL); + cur->fid = -1; + + return false; + } + + cur->slot = QUERY_IS_ASC_QUERY(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; + cur->fid = pQueryHandle->pFileGroup->fileId; + + STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; + STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; + SCompBlock* pBlock = pBlockInfo->pBlock.compBlock; + + return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); +} + static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { STsdbFileH* pFileHandle = tsdbGetFile(pQueryHandle->pTsdb); SQueryFilePos* cur = &pQueryHandle->cur; @@ -925,92 +967,12 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order); tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); - int32_t numOfBlocks = -1; - - // todo opt for only one table case - pQueryHandle->numOfBlocks = 0; - int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - - while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { - int32_t type = QUERY_IS_ASC_QUERY(pQueryHandle->order)? QUERY_RANGE_GREATER_EQUAL:QUERY_RANGE_LESS_EQUAL; - if (getFileCompInfo(pQueryHandle, &numOfBlocks, type) != TSDB_CODE_SUCCESS) { - break; - } - - assert(numOfBlocks >= 0); - dTrace("%p %d blocks found in file for %d table(s), fid:%d", pQueryHandle, numOfBlocks, - numOfTables, pQueryHandle->pFileGroup->fileId); - - // todo return error code to query engine - if (createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks) != TSDB_CODE_SUCCESS) { - break; - } - - assert(numOfBlocks >= pQueryHandle->numOfBlocks); - if (pQueryHandle->numOfBlocks > 0) { - break; - } - } - - // no data in file anymore - if (pQueryHandle->numOfBlocks <= 0) { - assert(pQueryHandle->pFileGroup == NULL); - cur->fid = -1; - - return false; - } - - cur->slot = QUERY_IS_ASC_QUERY(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; - cur->fid = pQueryHandle->pFileGroup->fileId; - - STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; - STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; - SCompBlock* pBlock = pBlockInfo->pBlock.compBlock; - - return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); + return getDataBlocksInFilesImpl(pQueryHandle); } else { if ((cur->slot == pQueryHandle->numOfBlocks - 1 && QUERY_IS_ASC_QUERY(pQueryHandle->order)) || (cur->slot == 0 && !QUERY_IS_ASC_QUERY(pQueryHandle->order))) { // all blocks - int32_t numOfBlocks = -1; - int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - pQueryHandle->numOfBlocks = 0; - - while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { - if (getFileCompInfo(pQueryHandle, &numOfBlocks, 1) != TSDB_CODE_SUCCESS) { - break; - } - - assert(numOfBlocks >= 0); - dTrace("%p %d blocks found in file for %d table(s), fid:%d", pQueryHandle, numOfBlocks, numOfTables, - pQueryHandle->pFileGroup->fileId); - - // todo return error code to query engine - if (createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks) != TSDB_CODE_SUCCESS) { - break; - } - - assert(numOfBlocks >= pQueryHandle->numOfBlocks); - if (pQueryHandle->numOfBlocks > 0) { - break; - } - } - - // no data in file anymore - if (pQueryHandle->numOfBlocks <= 0) { - assert(pQueryHandle->pFileGroup == NULL); - cur->fid = -1; - return false; - } - - cur->slot = QUERY_IS_ASC_QUERY(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; - cur->fid = pQueryHandle->pFileGroup->fileId; - - STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; - STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; - SCompBlock* pBlock = pBlockInfo->pBlock.compBlock; - - return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); + return getDataBlocksInFilesImpl(pQueryHandle); } else { // next block of the same file int32_t step = QUERY_IS_ASC_QUERY(pQueryHandle->order)? 1:-1; cur->slot += step; @@ -1027,11 +989,9 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { } } - static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); // todo add assert, the value of numOfTables should be less than the maximum value for each vnode capacity - assert(numOfTables > 0); while (pQueryHandle->activeIndex < numOfTables) { if (hasMoreDataInCache(pQueryHandle)) { @@ -1259,7 +1219,7 @@ static void convertQueryResult(SArray* pRes, SArray* pTableList) { } } -void destroyHelper(void* param) { +static void destroyHelper(void* param) { if (param == NULL) { return; } @@ -1269,86 +1229,6 @@ void destroyHelper(void* param) { free(param); } -static UNUSED_FUNC int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { - SPatternCompareInfo pInfo = {'%', '_'}; - - const char* pattern = pRight; - const char* str = pLeft; - - int32_t ret = patternMatch(pattern, str, strlen(str), &pInfo); - - return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; -} - -static UNUSED_FUNC int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { - SPatternCompareInfo pInfo = {'%', '_'}; - - const wchar_t* pattern = pRight; - const wchar_t* str = pLeft; - - int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo); - - return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; -} - -// static __compar_fn_t getFilterComparator(int32_t type, int32_t filterType, int32_t optr) { -// __compar_fn_t comparator = NULL; -// -// switch (type) { -// case TSDB_DATA_TYPE_TINYINT: -// case TSDB_DATA_TYPE_SMALLINT: -// case TSDB_DATA_TYPE_INT: -// case TSDB_DATA_TYPE_BIGINT: -// case TSDB_DATA_TYPE_BOOL: { -// if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { -// comparator = compareIntVal; -// } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { -// comparator = compareIntDoubleVal; -// } -// break; -// } -// -// case TSDB_DATA_TYPE_FLOAT: -// case TSDB_DATA_TYPE_DOUBLE: { -// if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { -// comparator = compareDoubleIntVal; -// } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { -// comparator = compareDoubleVal; -// } -// break; -// } -// -// case TSDB_DATA_TYPE_BINARY: { -// assert(filterType == TSDB_DATA_TYPE_BINARY); -// -// if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ -// comparator = compareStrPatternComp; -// } else { /* normal relational comparator */ -// comparator = compareStrVal; -// } -// -// break; -// } -// -// case TSDB_DATA_TYPE_NCHAR: { -// assert(filterType == TSDB_DATA_TYPE_NCHAR); -// -// if (optr == TSDB_RELATION_LIKE) { -// comparator = compareWStrPatternComp; -// } else { -// comparator = compareWStrVal; -// } -// -// break; -// } -// default: -// comparator = compareIntVal; -// break; -// } -// -// return comparator; -//} - static void getTagColumnInfo(SExprTreeSupporter* pSupporter, SSchema* pSchema, int32_t* index, int32_t* offset) { *index = 0; *offset = 0; From 5e58bd5c5f5702d82ce07507b01e5d3f8a646381 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 12 Apr 2020 22:38:19 +0800 Subject: [PATCH 66/78] remove potential race condition on FdObj --- src/rpc/src/rpcClient.c | 54 +++++++++++++++++--------------- src/rpc/src/rpcMain.c | 2 +- src/rpc/src/rpcServer.c | 68 ++++++++++++++++++++++------------------- 3 files changed, 67 insertions(+), 57 deletions(-) diff --git a/src/rpc/src/rpcClient.c b/src/rpc/src/rpcClient.c index a3105ef516..7ca24f6229 100644 --- a/src/rpc/src/rpcClient.c +++ b/src/rpc/src/rpcClient.c @@ -65,18 +65,18 @@ void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, pTcp->shandle = shandle; if (pthread_mutex_init(&(pTcp->mutex), NULL) < 0) { - tError("%s failed to init TCP mutex, reason:%s", label, strerror(errno)); + tError("%s failed to init TCP client mutex(%s)", label, strerror(errno)); return NULL; } if (pthread_cond_init(&(pTcp->fdReady), NULL) != 0) { - tError("%s init TCP condition variable failed, reason:%s\n", label, strerror(errno)); + tError("%s init TCP condition variable failed(%s)", label, strerror(errno)); return NULL; } pTcp->pollFd = epoll_create(10); // size does not matter if (pTcp->pollFd < 0) { - tError("%s failed to create TCP epoll", label); + tError("%s failed to create TCP client epoll", label); return NULL; } @@ -87,11 +87,11 @@ void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, int code = pthread_create(&(pTcp->thread), &thattr, taosReadTcpData, (void *)(pTcp)); pthread_attr_destroy(&thattr); if (code != 0) { - tError("%s failed to create TCP read data thread, reason:%s", label, strerror(errno)); + tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); return NULL; } - tTrace("%s TCP client is initialized, ip:%s port:%hu", label, ip, port); + tTrace("%s TCP client is initialized, ip:%s:%hu", label, ip, port); return pTcp; } @@ -181,15 +181,9 @@ int taosSendTcpClientData(uint32_t ip, uint16_t port, void *data, int len, void return (int)send(pFdObj->fd, data, (size_t)len, 0); } -static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { - STcpClient *pTcp; +static void taosReportBrokenLink(STcpFd *pFdObj) { SRecvInfo recvInfo; - - if (pFdObj == NULL) return; - if (pFdObj->signature != pFdObj) return; - - pFdObj->signature = NULL; - pTcp = pFdObj->pTcp; + STcpClient *pTcp = pFdObj->pTcp; if (pFdObj->thandle) { recvInfo.msg = NULL; @@ -202,6 +196,15 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { recvInfo.connType = RPC_CONN_TCP; (*(pTcp->processData))(&recvInfo); } +} + +static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { + + if (pFdObj == NULL) return; + if (pFdObj->signature != pFdObj) return; + + pFdObj->signature = NULL; + STcpClient *pTcp = pFdObj->pTcp; epoll_ctl(pTcp->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); close(pFdObj->fd); @@ -211,7 +214,7 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { pTcp->numOfFds--; if (pTcp->numOfFds < 0) - tError("%s number of TCP FDs shall never be negative, FD:%p", pTcp->label, pFdObj); + tError("%s %p, number of FDs is negative!!!, FD:%p", pTcp->label, pFdObj->thandle, pFdObj); if (pFdObj->prev) { (pFdObj->prev)->next = pFdObj->next; @@ -225,7 +228,7 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { pthread_mutex_unlock(&pTcp->mutex); - tTrace("%s TCP is cleaned up, FD:%p numOfFds:%d", pTcp->label, pFdObj, pTcp->numOfFds); + tTrace("%s %p, FD:%p is cleaned, numOfFds:%d", pTcp->label, pFdObj->thandle, pFdObj, pTcp->numOfFds); tfree(pFdObj); } @@ -250,29 +253,29 @@ static void *taosReadTcpData(void *param) { pFdObj = events[i].data.ptr; if (events[i].events & EPOLLERR) { - tTrace("%s TCP error happened on FD\n", pTcp->label); - taosCleanUpTcpFdObj(pFdObj); + tTrace("%s %p, TCP error happened on FD", pTcp->label, pFdObj->thandle); + taosReportBrokenLink(pFdObj); continue; } if (events[i].events & EPOLLHUP) { - tTrace("%s TCP FD hang up\n", pTcp->label); - taosCleanUpTcpFdObj(pFdObj); + tTrace("%s %p, TCP FD hang up", pTcp->label, pFdObj->thandle); + taosReportBrokenLink(pFdObj); continue; } int headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); if (headLen != sizeof(SRpcHead)) { - tError("%s read error, headLen:%d", pTcp->label, headLen); - taosCleanUpTcpFdObj(pFdObj); + tError("%s %p, read error, headLen:%d", pTcp->label, pFdObj->thandle, headLen); + taosReportBrokenLink(pFdObj); continue; } int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); char *buffer = (char *)malloc((size_t)msgLen + tsRpcOverhead); if (NULL == buffer) { - tTrace("%s TCP malloc(size:%d) fail\n", pTcp->label, msgLen); - taosCleanUpTcpFdObj(pFdObj); + tTrace("%s %p, TCP malloc(size:%d) fail", pTcp->label, pFdObj->thandle, msgLen); + taosReportBrokenLink(pFdObj); continue; } @@ -281,9 +284,10 @@ static void *taosReadTcpData(void *param) { int32_t retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); if (leftLen != retLen) { - tError("%s read error, leftLen:%d retLen:%d", pTcp->label, leftLen, retLen); + tError("%s %p, read error, leftLen:%d retLen:%d", + pTcp->label, pFdObj->thandle, leftLen, retLen); tfree(buffer); - taosCleanUpTcpFdObj(pFdObj); + taosReportBrokenLink(pFdObj); continue; } diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index bec9621e3b..8280264764 100755 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -817,7 +817,7 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) { SRpcInfo *pRpc = pConn->pRpc; tTrace("%s %p, link is broken", pRpc->label, pConn); - pConn->chandle = NULL; + // pConn->chandle = NULL; if (pConn->outType) { SRpcReqContext *pContext = pConn->pContext; diff --git a/src/rpc/src/rpcServer.c b/src/rpc/src/rpcServer.c index 7430157bcf..37576fa0f6 100644 --- a/src/rpc/src/rpcServer.c +++ b/src/rpc/src/rpcServer.c @@ -177,7 +177,6 @@ void taosCloseTcpServerConnection(void *chandle) { SFdObj *pFdObj = chandle; if (pFdObj == NULL) return; - pFdObj->thandle = NULL; taosCleanUpFdObj(pFdObj); } @@ -191,6 +190,25 @@ int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void #define maxEvents 10 +static void taosReportBrokenLink(SFdObj *pFdObj) { + + SThreadObj *pThreadObj = pFdObj->pThreadObj; + + // notify the upper layer, so it will clean the associated context + if (pFdObj->thandle) { + SRecvInfo recvInfo; + recvInfo.msg = NULL; + recvInfo.msgLen = 0; + recvInfo.ip = 0; + recvInfo.port = 0; + recvInfo.shandle = pThreadObj->shandle; + recvInfo.thandle = pFdObj->thandle;; + recvInfo.chandle = NULL; + recvInfo.connType = RPC_CONN_TCP; + (*(pThreadObj->processData))(&recvInfo); + } +} + static void taosProcessTcpData(void *param) { SThreadObj * pThreadObj; int i, fdNum; @@ -214,29 +232,29 @@ static void taosProcessTcpData(void *param) { pFdObj = events[i].data.ptr; if (events[i].events & EPOLLERR) { - tTrace("%s TCP thread:%d, error happened on FD", pThreadObj->label, pThreadObj->threadId); - taosCleanUpFdObj(pFdObj); + tTrace("%s %p, error happened on FD", pThreadObj->label, pFdObj->thandle); + taosReportBrokenLink(pFdObj); continue; } if (events[i].events & EPOLLHUP) { - tTrace("%s TCP thread:%d, FD hang up", pThreadObj->label, pThreadObj->threadId); - taosCleanUpFdObj(pFdObj); + tTrace("%s %p, FD hang up", pThreadObj->label, pFdObj->thandle); + taosReportBrokenLink(pFdObj); continue; } int32_t headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); if (headLen != sizeof(SRpcHead)) { - tError("%s read error, headLen:%d, errno:%d", pThreadObj->label, headLen, errno); - taosCleanUpFdObj(pFdObj); + tError("%s %p, read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen); + taosReportBrokenLink(pFdObj); continue; } int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); char *buffer = malloc(msgLen + tsRpcOverhead); if ( NULL == buffer) { - tError("%s TCP malloc(size:%d) fail\n", pThreadObj->label, msgLen); - taosCleanUpFdObj(pFdObj); + tError("%s %p, TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); + taosReportBrokenLink(pFdObj); continue; } @@ -245,8 +263,9 @@ static void taosProcessTcpData(void *param) { int32_t retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); if (leftLen != retLen) { - tError("%s read error, leftLen:%d retLen:%d", pThreadObj->label, leftLen, retLen); - taosCleanUpFdObj(pFdObj); + tError("%s %p, read error, leftLen:%d retLen:%d", + pThreadObj->label, pFdObj->thandle, leftLen, retLen); + taosReportBrokenLink(pFdObj); tfree(buffer); continue; } @@ -339,7 +358,7 @@ static void taosAcceptTcpConnection(void *arg) { pthread_cond_signal(&pThreadObj->fdReady); pthread_mutex_unlock(&(pThreadObj->threadMutex)); - tTrace("%s TCP thread:%d, a new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label, + tTrace("%s TCP thread:%d, new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label, pThreadObj->threadId, pFdObj->ipstr, pFdObj->port, pFdObj, pThreadObj->numOfFds); // pick up next thread for next connection @@ -349,27 +368,12 @@ static void taosAcceptTcpConnection(void *arg) { } static void taosCleanUpFdObj(SFdObj *pFdObj) { - SThreadObj *pThreadObj; if (pFdObj == NULL) return; if (pFdObj->signature != pFdObj) return; pFdObj->signature = NULL; - pThreadObj = pFdObj->pThreadObj; - - // notify the upper layer, so it will clean the associated context - if (pFdObj->thandle) { - SRecvInfo recvInfo; - recvInfo.msg = NULL; - recvInfo.msgLen = 0; - recvInfo.ip = 0; - recvInfo.port = 0; - recvInfo.shandle = pThreadObj->shandle; - recvInfo.thandle = pFdObj->thandle;; - recvInfo.chandle = NULL; - recvInfo.connType = RPC_CONN_TCP; - (*(pThreadObj->processData))(&recvInfo); - } + SThreadObj *pThreadObj = pFdObj->pThreadObj; close(pFdObj->fd); epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); @@ -379,7 +383,8 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { pThreadObj->numOfFds--; if (pThreadObj->numOfFds < 0) - tError("%s TCP thread:%d, number of FDs shall never be negative", pThreadObj->label, pThreadObj->threadId); + tError("%s %p, TCP thread:%d, number of FDs is negative!!!", + pThreadObj->label, pFdObj->thandle, pThreadObj->threadId); // remove from the FdObject list @@ -395,8 +400,9 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->threadMutex); - tTrace("%s TCP thread:%d, FD:%p is cleaned up, numOfFds:%d", pThreadObj->label, pThreadObj->threadId, - pFdObj, pThreadObj->numOfFds); + tTrace("%s %p, FD:%p is cleaned, numOfFds:%d", + pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); + tfree(pFdObj); } From d2e0f7de02140879668527edb9b62af4adea6cc4 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 12 Apr 2020 22:53:40 +0800 Subject: [PATCH 67/78] fix valgrind warning --- src/vnode/tsdb/src/tsdbMain.c | 10 +++++++++- src/vnode/tsdb/src/tsdbMeta.c | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 6873c69c2a..ee9413fc12 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -813,6 +813,13 @@ static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables) return iters; } +static void tsdbFreeMemTable(SMemTable *pMemTable) { + if (pMemTable) { + tSkipListDestroy(pMemTable->pData); + free(pMemTable); + } +} + // Commit to file static void *tsdbCommitData(void *arg) { // TODO @@ -859,7 +866,8 @@ static void *tsdbCommitData(void *arg) { // TODO: free the skiplist for (int i = 0; i < pCfg->maxTables; i++) { STable *pTable = pMeta->tables[i]; - if (pTable && pTable->imem) { // Here has memory leak + if (pTable && pTable->imem) { + tsdbFreeMemTable(pTable->imem); pTable->imem = NULL; } } diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index 0eb6dde1d0..0ab276c120 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -34,7 +34,7 @@ void *tsdbEncodeTable(STable *pTable, int *contLen) { *contLen = tsdbEstimateTableEncodeSize(pTable); if (*contLen < 0) return NULL; - void *ret = malloc(*contLen); + void *ret = calloc(1, *contLen); if (ret == NULL) return NULL; void *ptr = ret; From 593e7bb79ac360f61d1486471df339fb77095185 Mon Sep 17 00:00:00 2001 From: lihui Date: Mon, 13 Apr 2020 09:27:53 +0800 Subject: [PATCH 68/78] [modify webdoc] --- documentation/webdocs/markdowndocs/administrator-ch.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/webdocs/markdowndocs/administrator-ch.md b/documentation/webdocs/markdowndocs/administrator-ch.md index cbccc03cbb..289f2dbf91 100644 --- a/documentation/webdocs/markdowndocs/administrator-ch.md +++ b/documentation/webdocs/markdowndocs/administrator-ch.md @@ -105,7 +105,7 @@ RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接 **shellActivityTimer** - 默认值：3 -系统在服务端保持结果集的最长时间，范围[1-120]。 +系统在服务端保持结果集的最长时间，单位：秒，范围[1-120]。 **maxUsers** - 默认值：10,000 @@ -138,7 +138,7 @@ RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接 系统（服务端和客户端）运行日志开关： - 131 仅输出错误和警告信息 -- 135 输入错误（ERROR）、警告（WARN）、信息（Info） +- 135 输出错误（ERROR）、警告（WARN）、信息（Info） 不同应用场景的数据往往具有不同的数据特征，比如保留天数、副本数、采集频次、记录大小、采集点的数量、压缩等都可完全不同。为获得在存储上的最高效率，TDengine提供如下存储相关的系统配置参数： From 65f0ff6d89e47bb6d5dcddfbd382a3d8dacfdd2e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 13 Apr 2020 10:18:01 +0800 Subject: [PATCH 69/78] change tscPrint to tscTrace to avoid print debug info to shell console. --- src/client/src/tscServer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index ebac676b77..1d6f7d67e7 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -191,7 +191,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { } if (pSql->cmd.command < TSDB_SQL_MGMT) { - tscPrint("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port); + tscTrace("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port); memcpy(pMsg, pSql->cmd.payload + tsRpcHeadSize, pSql->cmd.payloadLen); SRpcMsg rpcMsg = { From 2589fc9efb826b9e93a1828a39be82f83488b03e Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 13 Apr 2020 16:09:31 +0800 Subject: [PATCH 70/78] [td-98] fix time range query bug when data kept in buffer --- src/query/inc/queryExecutor.h | 2 +- src/query/src/queryExecutor.c | 9 +++++-- src/vnode/tsdb/src/tsdbRead.c | 47 +++++++++++++++++++++++++++-------- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 3932650e1f..0302732995 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -160,7 +160,7 @@ typedef struct SQueryRuntimeEnv { SQueryCostSummary summary; bool stableQuery; // super table query or not void* pQueryHandle; - void* pSubQueryHandle; // another thread for + void* pSecQueryHandle; // another thread for SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file } SQueryRuntimeEnv; diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 0f74cf1beb..8015360919 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -1552,6 +1552,8 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { destroyResultBuf(pRuntimeEnv->pResultBuf); tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); + tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); + pRuntimeEnv->pTSBuf = tsBufDestory(pRuntimeEnv->pTSBuf); } @@ -2565,7 +2567,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order); - tsdb_query_handle_t pQueryHandle = pRuntimeEnv->scanFlag == MASTER_SCAN? pRuntimeEnv->pQueryHandle:pRuntimeEnv->pSubQueryHandle; + tsdb_query_handle_t pQueryHandle = pRuntimeEnv->scanFlag == MASTER_SCAN? pRuntimeEnv->pQueryHandle:pRuntimeEnv->pSecQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { @@ -3557,7 +3559,10 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { taosArrayPush(cols, &pQuery->colList[i]); } - pRuntimeEnv->pSubQueryHandle = tsdbQueryByTableId(pQInfo->tsdb, &cond, pQInfo->pTableIdList, cols); + if (pRuntimeEnv->pSecQueryHandle != NULL) { + pRuntimeEnv->pSecQueryHandle = tsdbQueryByTableId(pQInfo->tsdb, &cond, pQInfo->pTableIdList, cols); + } + taosArrayDestroy(cols); status = pQuery->status; diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 966debda8d..ee95199e5b 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -218,9 +218,31 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { if (pTable->mem == NULL && pTable->imem == NULL) { return false; } + + STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + pTable = pCheckInfo->pTableObj; + + if (pCheckInfo->iter == NULL) { + pCheckInfo->iter = tSkipListCreateIter(pTable->mem->pData); + if (pCheckInfo->iter == NULL) { + return false; + } + } + + if (!tSkipListIterNext(pCheckInfo->iter)) { // buffer is empty + return false; + } + SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); + if (node == NULL) { + return false; + } + + SDataRow row = SL_GET_NODE_DATA(node); + pCheckInfo->lastKey = dataRowKey(row); // first timestamp in buffer + // all data in mem are checked already. - if (pTableCheckInfo->lastKey > pTable->mem->keyLast) { + if (pTableCheckInfo->lastKey > pHandle->window.ekey) { return false; } @@ -522,13 +544,15 @@ static int vnodeBinarySearchKey(char* pValue, int num, TSKEY key, int order) { int numOfPoints; TSKEY* keyList; + assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + if (num <= 0) return -1; keyList = (TSKEY*)pValue; firstPos = 0; lastPos = num - 1; - if (order == 0) { + if (order == TSDB_ORDER_DESC) { // find the first position which is smaller than the key while (1) { if (key >= keyList[lastPos]) return lastPos; @@ -596,8 +620,8 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf pQueryHandle->realNumOfRows = cur->pos + 1; pCheckInfo->lastKey = blockInfo.window.ekey - 1; } else { - endPos = - vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfPoints, pQueryHandle->window.ekey, pQueryHandle->order); + int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; + endPos = vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfPoints, pQueryHandle->window.ekey, order); if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { if (endPos < cur->pos) { @@ -1042,7 +1066,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int max int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns); *skey = INT64_MIN; - while (tSkipListIterNext(pIter)) { + do/* (1) */{ SSkipListNode* node = tSkipListIterGet(pIter); if (node == NULL) break; @@ -1063,8 +1087,11 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int max } numOfRows++; - if (numOfRows >= maxRowsToRead) break; - }; + if (numOfRows >= maxRowsToRead) { + break; + } + + } while(tSkipListIterNext(pIter)); return numOfRows; } @@ -1107,10 +1134,8 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t* pQueryHandle) { if (pTable->mem != NULL) { // create mem table iterator if it is not created yet - if (pCheckInfo->iter == NULL) { - pCheckInfo->iter = tSkipListCreateIter(pTable->mem->pData); - } - rows = tsdbReadRowsFromCache(pCheckInfo->iter, INT64_MAX, 2, &skey, &ekey, pHandle); + assert(pCheckInfo->iter != NULL); + rows = tsdbReadRowsFromCache(pCheckInfo->iter, pHandle->window.ekey, 2, &skey, &ekey, pHandle); // update the last key value pCheckInfo->lastKey = ekey + 1; From 704d8da049519cfdc6e1da01e251e36e4254ddaf Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 13 Apr 2020 18:03:47 +0800 Subject: [PATCH 71/78] re-organize TCP connection code --- src/rpc/inc/rpcServer.h | 34 --- src/rpc/inc/{rpcClient.h => rpcTcp.h} | 12 +- src/rpc/src/rpcClient.c | 315 ------------------- src/rpc/src/rpcMain.c | 15 +- src/rpc/src/{rpcServer.c => rpcTcp.c} | 424 ++++++++++++-------------- 5 files changed, 212 insertions(+), 588 deletions(-) delete mode 100644 src/rpc/inc/rpcServer.h rename src/rpc/inc/{rpcClient.h => rpcTcp.h} (74%) delete mode 100644 src/rpc/src/rpcClient.c mode change 100755 => 100644 src/rpc/src/rpcMain.c rename src/rpc/src/{rpcServer.c => rpcTcp.c} (55%) diff --git a/src/rpc/inc/rpcServer.h b/src/rpc/inc/rpcServer.h deleted file mode 100644 index 6b238733a4..0000000000 --- a/src/rpc/inc/rpcServer.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _rpc_server_header_ -#define _rpc_server_header_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "taosdef.h" - -void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); -void taosCleanUpTcpServer(void *param); -void taosCloseTcpServerConnection(void *param); -int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void *chandle); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/rpc/inc/rpcClient.h b/src/rpc/inc/rpcTcp.h similarity index 74% rename from src/rpc/inc/rpcClient.h rename to src/rpc/inc/rpcTcp.h index c87ae79312..16972dbc7e 100644 --- a/src/rpc/inc/rpcClient.h +++ b/src/rpc/inc/rpcTcp.h @@ -13,20 +13,22 @@ * along with this program. If not, see . */ -#ifndef _rpc_client_header_ -#define _rpc_client_header_ +#ifndef _rpc_tcp_header_ +#define _rpc_tcp_header_ #ifdef __cplusplus extern "C" { #endif -#include "taosdef.h" +void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); +void taosCleanUpTcpServer(void *param); void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle); void taosCleanUpTcpClient(void *chandle); void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port); -void taosCloseTcpClientConnection(void *chandle); -int taosSendTcpClientData(uint32_t ip, uint16_t port, void *data, int len, void *chandle); + +void taosCloseTcpConnection(void *chandle); +int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle); #ifdef __cplusplus } diff --git a/src/rpc/src/rpcClient.c b/src/rpc/src/rpcClient.c deleted file mode 100644 index 7ca24f6229..0000000000 --- a/src/rpc/src/rpcClient.c +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "taosmsg.h" -#include "tlog.h" -#include "tsocket.h" -#include "tutil.h" -#include "rpcClient.h" -#include "rpcHead.h" - -#ifndef EPOLLWAKEUP -#define EPOLLWAKEUP (1u << 29) -#endif - -typedef struct _tcp_fd { - void *signature; - int fd; // TCP socket FD - void * thandle; - uint32_t ip; - char ipstr[20]; - uint16_t port; - struct _tcp_client *pTcp; - struct _tcp_fd * prev, *next; -} STcpFd; - -typedef struct _tcp_client { - pthread_t thread; - STcpFd * pHead; - pthread_mutex_t mutex; - pthread_cond_t fdReady; - int pollFd; - int numOfFds; - char label[12]; - char ipstr[20]; - void *shandle; // handle passed by upper layer during server initialization - void *(*processData)(SRecvInfo *pRecv); -} STcpClient; - -#define maxTcpEvents 100 - -static void taosCleanUpTcpFdObj(STcpFd *pFdObj); -static void *taosReadTcpData(void *param); - -void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle) { - STcpClient *pTcp; - pthread_attr_t thattr; - - pTcp = (STcpClient *)malloc(sizeof(STcpClient)); - memset(pTcp, 0, sizeof(STcpClient)); - strcpy(pTcp->label, label); - strcpy(pTcp->ipstr, ip); - pTcp->shandle = shandle; - - if (pthread_mutex_init(&(pTcp->mutex), NULL) < 0) { - tError("%s failed to init TCP client mutex(%s)", label, strerror(errno)); - return NULL; - } - - if (pthread_cond_init(&(pTcp->fdReady), NULL) != 0) { - tError("%s init TCP condition variable failed(%s)", label, strerror(errno)); - return NULL; - } - - pTcp->pollFd = epoll_create(10); // size does not matter - if (pTcp->pollFd < 0) { - tError("%s failed to create TCP client epoll", label); - return NULL; - } - - pTcp->processData = fp; - - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - int code = pthread_create(&(pTcp->thread), &thattr, taosReadTcpData, (void *)(pTcp)); - pthread_attr_destroy(&thattr); - if (code != 0) { - tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); - return NULL; - } - - tTrace("%s TCP client is initialized, ip:%s:%hu", label, ip, port); - - return pTcp; -} - -void taosCleanUpTcpClient(void *chandle) { - STcpClient *pTcp = (STcpClient *)chandle; - if (pTcp == NULL) return; - - while (pTcp->pHead) { - taosCleanUpTcpFdObj(pTcp->pHead); - pTcp->pHead = pTcp->pHead->next; - } - - close(pTcp->pollFd); - - pthread_cancel(pTcp->thread); - pthread_join(pTcp->thread, NULL); - - // tTrace (":%s, all connections are cleaned up", pTcp->label); - - tfree(pTcp); -} - -void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) { - STcpClient * pTcp = (STcpClient *)shandle; - STcpFd * pFdObj; - struct epoll_event event; - struct in_addr destIp; - int fd; - - fd = taosOpenTcpClientSocket(ip, port, pTcp->ipstr); - if (fd <= 0) return NULL; - - pFdObj = (STcpFd *)malloc(sizeof(STcpFd)); - if (pFdObj == NULL) { - tError("%s no enough resource to allocate TCP FD IDs", pTcp->label); - tclose(fd); - return NULL; - } - - memset(pFdObj, 0, sizeof(STcpFd)); - pFdObj->fd = fd; - strcpy(pFdObj->ipstr, ip); - inet_aton(ip, &destIp); - pFdObj->ip = destIp.s_addr; - pFdObj->port = port; - pFdObj->pTcp = pTcp; - pFdObj->thandle = thandle; - pFdObj->signature = pFdObj; - - event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; - event.data.ptr = pFdObj; - if (epoll_ctl(pTcp->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { - tError("%s failed to add TCP FD for epoll, error:%s", pTcp->label, strerror(errno)); - tfree(pFdObj); - tclose(fd); - return NULL; - } - - // notify the data process, add into the FdObj list - pthread_mutex_lock(&(pTcp->mutex)); - pFdObj->next = pTcp->pHead; - if (pTcp->pHead) (pTcp->pHead)->prev = pFdObj; - pTcp->pHead = pFdObj; - pTcp->numOfFds++; - pthread_cond_signal(&pTcp->fdReady); - pthread_mutex_unlock(&(pTcp->mutex)); - - tTrace("%s TCP connection to %s:%hu is created, FD:%p numOfFds:%d", pTcp->label, ip, port, pFdObj, pTcp->numOfFds); - - return pFdObj; -} - -void taosCloseTcpClientConnection(void *chandle) { - STcpFd *pFdObj = (STcpFd *)chandle; - - if (pFdObj == NULL) return; - - taosCleanUpTcpFdObj(pFdObj); -} - -int taosSendTcpClientData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { - STcpFd *pFdObj = (STcpFd *)chandle; - - if (chandle == NULL) return -1; - - return (int)send(pFdObj->fd, data, (size_t)len, 0); -} - -static void taosReportBrokenLink(STcpFd *pFdObj) { - SRecvInfo recvInfo; - STcpClient *pTcp = pFdObj->pTcp; - - if (pFdObj->thandle) { - recvInfo.msg = NULL; - recvInfo.msgLen = 0; - recvInfo.ip = 0; - recvInfo.port = 0; - recvInfo.shandle = pTcp->shandle; - recvInfo.thandle = pFdObj->thandle;; - recvInfo.chandle = NULL; - recvInfo.connType = RPC_CONN_TCP; - (*(pTcp->processData))(&recvInfo); - } -} - -static void taosCleanUpTcpFdObj(STcpFd *pFdObj) { - - if (pFdObj == NULL) return; - if (pFdObj->signature != pFdObj) return; - - pFdObj->signature = NULL; - STcpClient *pTcp = pFdObj->pTcp; - - epoll_ctl(pTcp->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); - close(pFdObj->fd); - - pthread_mutex_lock(&pTcp->mutex); - - pTcp->numOfFds--; - - if (pTcp->numOfFds < 0) - tError("%s %p, number of FDs is negative!!!, FD:%p", pTcp->label, pFdObj->thandle, pFdObj); - - if (pFdObj->prev) { - (pFdObj->prev)->next = pFdObj->next; - } else { - pTcp->pHead = pFdObj->next; - } - - if (pFdObj->next) { - (pFdObj->next)->prev = pFdObj->prev; - } - - pthread_mutex_unlock(&pTcp->mutex); - - tTrace("%s %p, FD:%p is cleaned, numOfFds:%d", pTcp->label, pFdObj->thandle, pFdObj, pTcp->numOfFds); - - tfree(pFdObj); -} - -static void *taosReadTcpData(void *param) { - STcpClient *pTcp = (STcpClient *)param; - int i, fdNum; - STcpFd *pFdObj; - struct epoll_event events[maxTcpEvents]; - SRecvInfo recvInfo; - SRpcHead rpcHead; - - while (1) { - pthread_mutex_lock(&pTcp->mutex); - if (pTcp->numOfFds < 1) pthread_cond_wait(&pTcp->fdReady, &pTcp->mutex); - pthread_mutex_unlock(&pTcp->mutex); - - fdNum = epoll_wait(pTcp->pollFd, events, maxTcpEvents, -1); - if (fdNum < 0) continue; - - for (i = 0; i < fdNum; ++i) { - pFdObj = events[i].data.ptr; - - if (events[i].events & EPOLLERR) { - tTrace("%s %p, TCP error happened on FD", pTcp->label, pFdObj->thandle); - taosReportBrokenLink(pFdObj); - continue; - } - - if (events[i].events & EPOLLHUP) { - tTrace("%s %p, TCP FD hang up", pTcp->label, pFdObj->thandle); - taosReportBrokenLink(pFdObj); - continue; - } - - int headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); - if (headLen != sizeof(SRpcHead)) { - tError("%s %p, read error, headLen:%d", pTcp->label, pFdObj->thandle, headLen); - taosReportBrokenLink(pFdObj); - continue; - } - - int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); - char *buffer = (char *)malloc((size_t)msgLen + tsRpcOverhead); - if (NULL == buffer) { - tTrace("%s %p, TCP malloc(size:%d) fail", pTcp->label, pFdObj->thandle, msgLen); - taosReportBrokenLink(pFdObj); - continue; - } - - char *msg = buffer + tsRpcOverhead; - int32_t leftLen = msgLen - headLen; - int32_t retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); - - if (leftLen != retLen) { - tError("%s %p, read error, leftLen:%d retLen:%d", - pTcp->label, pFdObj->thandle, leftLen, retLen); - tfree(buffer); - taosReportBrokenLink(pFdObj); - continue; - } - - // tTrace("%s TCP data is received, ip:%s:%u len:%d", pTcp->label, pFdObj->ipstr, pFdObj->port, msgLen); - - memcpy(msg, &rpcHead, sizeof(SRpcHead)); - recvInfo.msg = msg; - recvInfo.msgLen = msgLen; - recvInfo.ip = pFdObj->ip; - recvInfo.port = pFdObj->port; - recvInfo.shandle = pTcp->shandle; - recvInfo.thandle = pFdObj->thandle;; - recvInfo.chandle = pFdObj; - recvInfo.connType = RPC_CONN_TCP; - - pFdObj->thandle = (*(pTcp->processData))(&recvInfo); - - if (pFdObj->thandle == NULL) taosCleanUpTcpFdObj(pFdObj); - } - } - - return NULL; -} - - diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c old mode 100755 new mode 100644 index 8280264764..5048d5db14 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -27,8 +27,7 @@ #include "taosmsg.h" #include "rpcUdp.h" #include "rpcCache.h" -#include "rpcClient.h" -#include "rpcServer.h" +#include "rpcTcp.h" #include "rpcHead.h" #include "trpc.h" #include "hash.h" @@ -67,7 +66,7 @@ typedef struct { void *udphandle;// returned handle from UDP initialization void *pCache; // connection cache pthread_mutex_t mutex; - struct _RpcConn *connList; // connection list + struct SRpcConn *connList; // connection list } SRpcInfo; typedef struct { @@ -88,7 +87,7 @@ typedef struct { char msg[0]; // RpcHead starts from here } SRpcReqContext; -typedef struct _RpcConn { +typedef struct SRpcConn { int sid; // session ID uint32_t ownId; // own link ID uint32_t peerId; // peer link ID @@ -156,8 +155,8 @@ void (*taosCleanUpConn[])(void *thandle) = { int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = { taosSendUdpData, taosSendUdpData, - taosSendTcpServerData, - taosSendTcpClientData + taosSendTcpData, + taosSendTcpData }; void *(*taosOpenConn[])(void *shandle, void *thandle, char *ip, uint16_t port) = { @@ -170,8 +169,8 @@ void *(*taosOpenConn[])(void *shandle, void *thandle, char *ip, uint16_t port) = void (*taosCloseConn[])(void *chandle) = { NULL, NULL, - taosCloseTcpServerConnection, - taosCloseTcpClientConnection + taosCloseTcpConnection, + taosCloseTcpConnection }; static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort, int8_t connType); diff --git a/src/rpc/src/rpcServer.c b/src/rpc/src/rpcTcp.c similarity index 55% rename from src/rpc/src/rpcServer.c rename to src/rpc/src/rpcTcp.c index 37576fa0f6..27b81deda5 100644 --- a/src/rpc/src/rpcServer.c +++ b/src/rpc/src/rpcTcp.c @@ -18,30 +18,30 @@ #include "tlog.h" #include "tsocket.h" #include "tutil.h" -#include "rpcServer.h" #include "rpcHead.h" +#include "rpcTcp.h" -#define TAOS_IPv4ADDR_LEN 16 #ifndef EPOLLWAKEUP #define EPOLLWAKEUP (1u << 29) #endif -typedef struct _fd_obj { - void *signature; - int fd; // TCP socket FD - void * thandle; // handle from upper layer, like TAOS - char ipstr[TAOS_IPv4ADDR_LEN]; - unsigned int ip; - uint16_t port; - struct _thread_obj *pThreadObj; - struct _fd_obj * prev, *next; +typedef struct SFdObj { + void *signature; + int fd; // TCP socket FD + void *thandle; // handle from upper layer, like TAOS + uint32_t ip; + uint16_t port; + struct SThreadObj *pThreadObj; + struct SFdObj *prev; + struct SFdObj *next; } SFdObj; -typedef struct _thread_obj { +typedef struct SThreadObj { pthread_t thread; SFdObj * pHead; - pthread_mutex_t threadMutex; + pthread_mutex_t mutex; pthread_cond_t fdReady; + char ipstr[TSDB_IPv4ADDR_LEN]; int pollFd; int numOfFds; int threadId; @@ -51,7 +51,7 @@ typedef struct _thread_obj { } SThreadObj; typedef struct { - char ip[40]; + char ip[TSDB_IPv4ADDR_LEN]; uint16_t port; char label[12]; int numOfThreads; @@ -60,13 +60,15 @@ typedef struct { pthread_t thread; } SServerObj; -static void taosCleanUpFdObj(SFdObj *pFdObj); -static void taosProcessTcpData(void *param); -static void taosAcceptTcpConnection(void *arg); +static void *taosProcessTcpData(void *param); +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int fd); +static void taosFreeFdObj(SFdObj *pFdObj); +static void taosReportBrokenLink(SFdObj *pFdObj); +static void taosAcceptTcpConnection(void *arg); void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { - SServerObj *pServerObj; - SThreadObj *pThreadObj; + SServerObj *pServerObj; + SThreadObj *pThreadObj; pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1); strcpy(pServerObj->ip, ip); @@ -88,7 +90,7 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, strcpy(pThreadObj->label, label); pThreadObj->shandle = shandle; - code = pthread_mutex_init(&(pThreadObj->threadMutex), NULL); + code = pthread_mutex_init(&(pThreadObj->mutex), NULL); if (code < 0) { tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); break;; @@ -110,7 +112,7 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - code = pthread_create(&(pThreadObj->thread), &thattr, (void *)taosProcessTcpData, (void *)(pThreadObj)); + code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); pthread_attr_destroy(&thattr); if (code != 0) { tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); @@ -144,8 +146,8 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, } void taosCleanUpTcpServer(void *handle) { - SThreadObj *pThreadObj; SServerObj *pServerObj = handle; + SThreadObj *pThreadObj; if (pServerObj == NULL) return; @@ -156,7 +158,7 @@ void taosCleanUpTcpServer(void *handle) { pThreadObj = pServerObj->pThreadObj + i; while (pThreadObj->pHead) { - taosCleanUpFdObj(pThreadObj->pHead); + taosFreeFdObj(pThreadObj->pHead); pThreadObj->pHead = pThreadObj->pHead; } @@ -164,7 +166,7 @@ void taosCleanUpTcpServer(void *handle) { pthread_cancel(pThreadObj->thread); pthread_join(pThreadObj->thread, NULL); pthread_cond_destroy(&(pThreadObj->fdReady)); - pthread_mutex_destroy(&(pThreadObj->threadMutex)); + pthread_mutex_destroy(&(pThreadObj->mutex)); } tTrace("TCP:%s, TCP server is cleaned up", pServerObj->label); @@ -173,14 +175,146 @@ void taosCleanUpTcpServer(void *handle) { tfree(pServerObj); } -void taosCloseTcpServerConnection(void *chandle) { +static void taosAcceptTcpConnection(void *arg) { + int connFd = -1; + struct sockaddr_in caddr; + int sockFd; + int threadId = 0; + SThreadObj *pThreadObj; + SServerObj *pServerObj; + + pServerObj = (SServerObj *)arg; + + sockFd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port); + if (sockFd < 0) return; + + tTrace("%s TCP server is ready, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); + + while (1) { + socklen_t addrlen = sizeof(caddr); + connFd = accept(sockFd, (struct sockaddr *)&caddr, &addrlen); + + if (connFd < 0) { + tError("%s TCP accept failure(%s)", pServerObj->label, errno, strerror(errno)); + continue; + } + + tTrace("%s TCP connection from ip:%s:%hu", pServerObj->label, inet_ntoa(caddr.sin_addr), caddr.sin_port); + taosKeepTcpAlive(connFd); + + // pick up the thread to handle this connection + pThreadObj = pServerObj->pThreadObj + threadId; + + SFdObj *pFdObj = taosMallocFdObj(pThreadObj, connFd); + if (pFdObj) { + pFdObj->ip = caddr.sin_addr.s_addr; + pFdObj->port = caddr.sin_port; + tTrace("%s new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label, + inet_ntoa(caddr.sin_addr), pFdObj->port, pFdObj, pThreadObj->numOfFds); + } else { + close(connFd); + tError("%s failed to malloc FdObj(%s)", pServerObj->label, strerror(errno)); + } + + // pick up next thread for next connection + threadId++; + threadId = threadId % pServerObj->numOfThreads; + } +} + +void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle) { + SThreadObj *pThreadObj; + pthread_attr_t thattr; + + pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj)); + memset(pThreadObj, 0, sizeof(SThreadObj)); + strcpy(pThreadObj->label, label); + strcpy(pThreadObj->ipstr, ip); + pThreadObj->shandle = shandle; + + if (pthread_mutex_init(&(pThreadObj->mutex), NULL) < 0) { + tError("%s failed to init TCP client mutex(%s)", label, strerror(errno)); + return NULL; + } + + if (pthread_cond_init(&(pThreadObj->fdReady), NULL) != 0) { + tError("%s init TCP condition variable failed(%s)", label, strerror(errno)); + return NULL; + } + + pThreadObj->pollFd = epoll_create(10); // size does not matter + if (pThreadObj->pollFd < 0) { + tError("%s failed to create TCP client epoll", label); + return NULL; + } + + pThreadObj->processData = fp; + + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + int code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); + pthread_attr_destroy(&thattr); + if (code != 0) { + tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); + return NULL; + } + + tTrace("%s TCP client is initialized, ip:%s:%hu", label, ip, port); + + return pThreadObj; +} + +void taosCleanUpTcpClient(void *chandle) { + SThreadObj *pThreadObj = chandle; + if (pThreadObj == NULL) return; + + while (pThreadObj->pHead) { + taosFreeFdObj(pThreadObj->pHead); + pThreadObj->pHead = pThreadObj->pHead->next; + } + + close(pThreadObj->pollFd); + + pthread_cancel(pThreadObj->thread); + pthread_join(pThreadObj->thread, NULL); + + tTrace (":%s, all connections are cleaned up", pThreadObj->label); + + tfree(pThreadObj); +} + +void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) { + SThreadObj * pThreadObj = shandle; + struct in_addr destIp; + + int fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ipstr); + if (fd <= 0) return NULL; + + inet_aton(ip, &destIp); + SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd); + + if (pFdObj) { + pFdObj->thandle = thandle; + pFdObj->port = port; + pFdObj->ip = destIp.s_addr; + tTrace("%s %p, TCP connection to %s:%hu is created, FD:%p numOfFds:%d", + pThreadObj->label, thandle, ip, port, pFdObj, pThreadObj->numOfFds); + } else { + close(fd); + tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); + } + + return pFdObj; +} + +void taosCloseTcpConnection(void *chandle) { SFdObj *pFdObj = chandle; if (pFdObj == NULL) return; - taosCleanUpFdObj(pFdObj); + taosFreeFdObj(pFdObj); } -int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { +int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { SFdObj *pFdObj = chandle; if (chandle == NULL) return -1; @@ -188,8 +322,6 @@ int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void return (int)send(pFdObj->fd, data, (size_t)len, 0); } -#define maxEvents 10 - static void taosReportBrokenLink(SFdObj *pFdObj) { SThreadObj *pThreadObj = pFdObj->pThreadObj; @@ -209,26 +341,26 @@ static void taosReportBrokenLink(SFdObj *pFdObj) { } } -static void taosProcessTcpData(void *param) { - SThreadObj * pThreadObj; - int i, fdNum; - SFdObj * pFdObj; +#define maxEvents 10 + +static void *taosProcessTcpData(void *param) { + SThreadObj *pThreadObj = param; + SFdObj *pFdObj; struct epoll_event events[maxEvents]; SRecvInfo recvInfo; - pThreadObj = (SThreadObj *)param; SRpcHead rpcHead; while (1) { - pthread_mutex_lock(&pThreadObj->threadMutex); + pthread_mutex_lock(&pThreadObj->mutex); if (pThreadObj->numOfFds < 1) { - pthread_cond_wait(&pThreadObj->fdReady, &pThreadObj->threadMutex); + pthread_cond_wait(&pThreadObj->fdReady, &pThreadObj->mutex); } - pthread_mutex_unlock(&pThreadObj->threadMutex); + pthread_mutex_unlock(&pThreadObj->mutex); - fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, -1); + int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, -1); if (fdNum < 0) continue; - for (i = 0; i < fdNum; ++i) { + for (int i = 0; i < fdNum; ++i) { pFdObj = events[i].data.ptr; if (events[i].events & EPOLLERR) { @@ -270,7 +402,7 @@ static void taosProcessTcpData(void *param) { continue; } - // tTrace("%s TCP data is received, ip:%s:%u len:%d", pTcp->label, pFdObj->ipstr, pFdObj->port, msgLen); + // tTrace("%s TCP data is received, ip:%s:%u len:%d", pThreadObj->label, pFdObj->ipstr, pFdObj->port, msgLen); memcpy(msg, &rpcHead, sizeof(SRpcHead)); recvInfo.msg = msg; @@ -283,91 +415,43 @@ static void taosProcessTcpData(void *param) { recvInfo.connType = RPC_CONN_TCP; pFdObj->thandle = (*(pThreadObj->processData))(&recvInfo); - if (pFdObj->thandle == NULL) taosCleanUpFdObj(pFdObj); + if (pFdObj->thandle == NULL) taosFreeFdObj(pFdObj); } } + + return NULL; } -static void taosAcceptTcpConnection(void *arg) { - int connFd = -1; - struct sockaddr_in clientAddr; - int sockFd; - int threadId = 0; - SThreadObj * pThreadObj; - SServerObj * pServerObj; - SFdObj * pFdObj; +static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int fd) { struct epoll_event event; - pServerObj = (SServerObj *)arg; + SFdObj *pFdObj = (SFdObj *)calloc(sizeof(SFdObj), 1); + if (pFdObj == NULL) return NULL; - sockFd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port); + pFdObj->fd = fd; + pFdObj->pThreadObj = pThreadObj; + pFdObj->signature = pFdObj; - if (sockFd < 0) { - tError("%s failed to open TCP socket, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); - return; - } else { - tTrace("%s TCP server is ready, ip:%s:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); + event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; + event.data.ptr = pFdObj; + if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { + tfree(pFdObj); + return NULL; } - while (1) { - socklen_t addrlen = sizeof(clientAddr); - connFd = accept(sockFd, (struct sockaddr *)&clientAddr, &addrlen); + // notify the data process, add into the FdObj list + pthread_mutex_lock(&(pThreadObj->mutex)); + pFdObj->next = pThreadObj->pHead; + if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj; + pThreadObj->pHead = pFdObj; + pThreadObj->numOfFds++; + pthread_cond_signal(&pThreadObj->fdReady); + pthread_mutex_unlock(&(pThreadObj->mutex)); - if (connFd < 0) { - tError("%s TCP accept failure(%s)", pServerObj->label, errno, strerror(errno)); - continue; - } - - tTrace("%s TCP connection from ip:%s:%hu", pServerObj->label, inet_ntoa(clientAddr.sin_addr), - htons(clientAddr.sin_port)); - taosKeepTcpAlive(connFd); - - // pick up the thread to handle this connection - pThreadObj = pServerObj->pThreadObj + threadId; - - pFdObj = (SFdObj *)malloc(sizeof(SFdObj)); - if (pFdObj == NULL) { - tError("%s no enough resource to allocate TCP FD IDs", pServerObj->label); - close(connFd); - continue; - } - - memset(pFdObj, 0, sizeof(SFdObj)); - pFdObj->fd = connFd; - strcpy(pFdObj->ipstr, inet_ntoa(clientAddr.sin_addr)); - pFdObj->ip = clientAddr.sin_addr.s_addr; - pFdObj->port = htons(clientAddr.sin_port); - pFdObj->pThreadObj = pThreadObj; - pFdObj->signature = pFdObj; - - event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; - event.data.ptr = pFdObj; - if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { - tError("%s failed to add TCP FD for epoll(%s)", pServerObj->label, strerror(errno)); - tfree(pFdObj); - close(connFd); - continue; - } - - // notify the data process, add into the FdObj list - pthread_mutex_lock(&(pThreadObj->threadMutex)); - pFdObj->next = pThreadObj->pHead; - if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj; - pThreadObj->pHead = pFdObj; - pThreadObj->numOfFds++; - pthread_cond_signal(&pThreadObj->fdReady); - pthread_mutex_unlock(&(pThreadObj->threadMutex)); - - tTrace("%s TCP thread:%d, new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label, - pThreadObj->threadId, pFdObj->ipstr, pFdObj->port, pFdObj, pThreadObj->numOfFds); - - // pick up next thread for next connection - threadId++; - threadId = threadId % pServerObj->numOfThreads; - } + return pFdObj; } -static void taosCleanUpFdObj(SFdObj *pFdObj) { +static void taosFreeFdObj(SFdObj *pFdObj) { if (pFdObj == NULL) return; if (pFdObj->signature != pFdObj) return; @@ -378,7 +462,7 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { close(pFdObj->fd); epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL); - pthread_mutex_lock(&pThreadObj->threadMutex); + pthread_mutex_lock(&pThreadObj->mutex); pThreadObj->numOfFds--; @@ -398,7 +482,7 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { (pFdObj->next)->prev = pFdObj->prev; } - pthread_mutex_unlock(&pThreadObj->threadMutex); + pthread_mutex_unlock(&pThreadObj->mutex); tTrace("%s %p, FD:%p is cleaned, numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); @@ -406,116 +490,4 @@ static void taosCleanUpFdObj(SFdObj *pFdObj) { tfree(pFdObj); } -#if 0 -static void taosAcceptUDConnection(void *arg) { - int connFd = -1; - int sockFd; - int threadId = 0; - SThreadObj * pThreadObj; - SServerObj * pServerObj; - SFdObj * pFdObj; - struct epoll_event event; - - pServerObj = (SServerObj *)arg; - sockFd = taosOpenUDServerSocket(pServerObj->ip, pServerObj->port); - - if (sockFd < 0) { - tError("%s failed to open UD socket, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); - return; - } else { - tTrace("%s UD server is ready, ip:%s, port:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); - } - - while (1) { - connFd = accept(sockFd, NULL, NULL); - - if (connFd < 0) { - tError("%s UD accept failure, errno:%d, reason:%s", pServerObj->label, errno, strerror(errno)); - continue; - } - - // pick up the thread to handle this connection - pThreadObj = pServerObj->pThreadObj + threadId; - - pFdObj = (SFdObj *)malloc(sizeof(SFdObj)); - if (pFdObj == NULL) { - tError("%s no enough resource to allocate TCP FD IDs", pServerObj->label); - close(connFd); - continue; - } - - memset(pFdObj, 0, sizeof(SFdObj)); - pFdObj->fd = connFd; - pFdObj->pThreadObj = pThreadObj; - - event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP; - event.data.ptr = pFdObj; - if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { - tError("%s failed to add UD FD for epoll, error:%s", pServerObj->label, strerror(errno)); - tfree(pFdObj); - close(connFd); - continue; - } - - // notify the data process, add into the FdObj list - pthread_mutex_lock(&(pThreadObj->threadMutex)); - pFdObj->next = pThreadObj->pHead; - if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj; - pThreadObj->pHead = pFdObj; - pThreadObj->numOfFds++; - pthread_cond_signal(&pThreadObj->fdReady); - pthread_mutex_unlock(&(pThreadObj->threadMutex)); - - tTrace("%s UD thread:%d, a new connection, numOfFds:%d", pServerObj->label, pThreadObj->threadId, - pThreadObj->numOfFds); - - // pick up next thread for next connection - threadId++; - threadId = threadId % pServerObj->numOfThreads; - } -} -#endif - -#if 0 -void taosListTcpConnection(void *handle, char *buffer) { - SServerObj *pServerObj; - SThreadObj *pThreadObj; - SFdObj * pFdObj; - int i, numOfFds, numOfConns; - char * msg; - - pServerObj = (SServerObj *)handle; - buffer[0] = 0; - msg = buffer; - numOfConns = 0; - - pThreadObj = pServerObj->pThreadObj; - - for (i = 0; i < pServerObj->numOfThreads; ++i) { - numOfFds = 0; - sprintf(msg, "TCP:%s Thread:%d number of connections:%d\n", pServerObj->label, pThreadObj->threadId, - pThreadObj->numOfFds); - msg = msg + strlen(msg); - pFdObj = pThreadObj->pHead; - while (pFdObj) { - sprintf(msg, " ip:%s port:%hu\n", pFdObj->ipstr, pFdObj->port); - msg = msg + strlen(msg); - numOfFds++; - numOfConns++; - pFdObj = pFdObj->next; - } - - if (numOfFds != pThreadObj->numOfFds) - tError("TCP:%s thread:%d BIG error, numOfFds:%d actual numOfFds:%d", pServerObj->label, pThreadObj->threadId, - pThreadObj->numOfFds, numOfFds); - - pThreadObj++; - } - - sprintf(msg, "TCP:%s total connections:%d\n", pServerObj->label, numOfConns); - - return; -} -#endif - From e01081f2755596a1822bd4ea1d2cda189bb40468 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 13 Apr 2020 18:50:30 +0800 Subject: [PATCH 72/78] rename some structures --- src/rpc/src/rpcCache.c | 10 +++++----- src/rpc/src/rpcHaship.c | 14 +++++++------- src/rpc/src/rpcUdp.c | 24 ++++++++++++------------ 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/rpc/src/rpcCache.c b/src/rpc/src/rpcCache.c index a4863ef61d..3370b31277 100644 --- a/src/rpc/src/rpcCache.c +++ b/src/rpc/src/rpcCache.c @@ -22,18 +22,18 @@ #include "tutil.h" #include "rpcCache.h" -typedef struct _c_hash_t { +typedef struct SConnHash { uint32_t ip; uint16_t port; char connType; - struct _c_hash_t *prev; - struct _c_hash_t *next; - void * data; + struct SConnHash *prev; + struct SConnHash *next; + void *data; uint64_t time; } SConnHash; typedef struct { - SConnHash ** connHashList; + SConnHash **connHashList; mpool_h connHashMemPool; int maxSessions; int total; diff --git a/src/rpc/src/rpcHaship.c b/src/rpc/src/rpcHaship.c index c904dba4a1..43f2d138de 100644 --- a/src/rpc/src/rpcHaship.c +++ b/src/rpc/src/rpcHaship.c @@ -17,13 +17,13 @@ #include "tlog.h" #include "tmempool.h" -typedef struct _ip_hash_t { +typedef struct SIpHash { uint32_t ip; uint16_t port; int hash; - struct _ip_hash_t *prev; - struct _ip_hash_t *next; - void * data; + struct SIpHash *prev; + struct SIpHash *next; + void *data; } SIpHash; typedef struct { @@ -47,7 +47,7 @@ int rpcHashIp(void *handle, uint32_t ip, uint16_t port) { void *rpcAddIpHash(void *handle, void *data, uint32_t ip, uint16_t port) { int hash; - SIpHash * pNode; + SIpHash *pNode; SHashObj *pObj; pObj = (SHashObj *)handle; @@ -70,7 +70,7 @@ void *rpcAddIpHash(void *handle, void *data, uint32_t ip, uint16_t port) { void rpcDeleteIpHash(void *handle, uint32_t ip, uint16_t port) { int hash; - SIpHash * pNode; + SIpHash *pNode; SHashObj *pObj; pObj = (SHashObj *)handle; @@ -102,7 +102,7 @@ void rpcDeleteIpHash(void *handle, uint32_t ip, uint16_t port) { void *rpcGetIpHash(void *handle, uint32_t ip, uint16_t port) { int hash; - SIpHash * pNode; + SIpHash *pNode; SHashObj *pObj; pObj = (SHashObj *)handle; diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index e666187cf1..1e5ac3c6b5 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -32,7 +32,7 @@ int tsUdpDelay = 0; typedef struct { - void * signature; + void *signature; int index; int fd; uint16_t port; // peer port @@ -53,23 +53,23 @@ typedef struct { int server; char ip[16]; // local IP uint16_t port; // local Port - void * shandle; // handle passed by upper layer during server initialization + void *shandle; // handle passed by upper layer during server initialization int threads; char label[12]; - void * tmrCtrl; + void *tmrCtrl; void *(*fp)(SRecvInfo *pPacket); SUdpConn udpConn[]; } SUdpConnSet; typedef struct { - void * signature; + void *signature; uint32_t ip; // dest IP uint16_t port; // dest Port - SUdpConn * pConn; + SUdpConn *pConn; struct sockaddr_in destAdd; - void * msgHdr; + void *msgHdr; int totalLen; - void * timer; + void *timer; int emptyNum; } SUdpBuf; @@ -78,8 +78,8 @@ static SUdpBuf *taosCreateUdpBuf(SUdpConn *pConn, uint32_t ip, uint16_t port); static void taosProcessUdpBufTimer(void *param, void *tmrId); void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, void *fp, void *shandle) { - SUdpConn * pConn; - SUdpConnSet * pSet; + SUdpConn *pConn; + SUdpConnSet *pSet; int size = (int)sizeof(SUdpConnSet) + threads * (int)sizeof(SUdpConn); pSet = (SUdpConnSet *)malloc((size_t)size); @@ -164,7 +164,7 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v void taosCleanUpUdpConnection(void *handle) { SUdpConnSet *pSet = (SUdpConnSet *)handle; - SUdpConn * pConn; + SUdpConn *pConn; if (pSet == NULL) return; @@ -205,10 +205,10 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, char *ip, uint16_t por } static void *taosRecvUdpData(void *param) { + SUdpConn *pConn = param; struct sockaddr_in sourceAdd; int dataLen; unsigned int addLen; - SUdpConn * pConn = (SUdpConn *)param; uint16_t port; int minSize = sizeof(SRpcHead); SRecvInfo recvInfo; @@ -274,7 +274,7 @@ static void *taosRecvUdpData(void *param) { int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle) { SUdpConn *pConn = (SUdpConn *)chandle; - SUdpBuf * pBuf; + SUdpBuf *pBuf; if (pConn == NULL || pConn->signature != pConn) return -1; From f2b41206ce816b3e751b89fca931058aaa271bb4 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 13 Apr 2020 19:25:08 +0800 Subject: [PATCH 73/78] rename a structure --- src/dnode/src/dnodeWrite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnode/src/dnodeWrite.c b/src/dnode/src/dnodeWrite.c index b56e0d8ad7..7eb4add831 100644 --- a/src/dnode/src/dnodeWrite.c +++ b/src/dnode/src/dnodeWrite.c @@ -41,7 +41,7 @@ typedef struct { SRpcMsg rpcMsg; } SWriteMsg; -typedef struct _wworker_pool { +typedef struct { int32_t max; // max number of workers int32_t nextId; // from 0 to max-1, cyclic SWriteWorker *writeWorker; From 34a3ac2436bab39f40261381d5aafa0d66ba0a94 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 13 Apr 2020 19:25:38 +0800 Subject: [PATCH 74/78] [TD-52] make sdb use wal --- src/dnode/src/dnodeMClient.c | 95 +- src/mnode/CMakeLists.txt | 2 +- src/mnode/inc/mgmtSdb.h | 19 +- src/mnode/src/mgmtDb.c | 19 +- src/mnode/src/mgmtMain.c | 8 +- src/mnode/src/mgmtSdb.c | 984 +++++-------- src/mnode/src/mgmtTable.c | 68 +- src/mnode/src/mgmtUser.c | 29 +- src/mnode/src/mgmtVgroup.c | 17 +- src/util/inc/cJSON.h | 3 + src/util/src/cJSON.c | 3 + tests/tsim/inc/cJSON.h | 267 ---- tests/tsim/src/cJSON.c | 2702 ---------------------------------- 13 files changed, 531 insertions(+), 3685 deletions(-) delete mode 100644 tests/tsim/inc/cJSON.h delete mode 100644 tests/tsim/src/cJSON.c diff --git a/src/dnode/src/dnodeMClient.c b/src/dnode/src/dnodeMClient.c index c09583bd65..f9794d040b 100644 --- a/src/dnode/src/dnodeMClient.c +++ b/src/dnode/src/dnodeMClient.c @@ -34,27 +34,26 @@ static void dnodeProcessRspFromMnode(SRpcMsg *pMsg); static void dnodeProcessStatusRsp(SRpcMsg *pMsg); static void (*tsDnodeProcessMgmtRspFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); static void *tsDnodeMClientRpc = NULL; -static SRpcIpSet tsMpeerIpList = {0}; -static SDMNodeInfos tsMpeerInfos = {0}; +static SRpcIpSet tsMnodeIpList = {0}; +static SDMNodeInfos tsMnodeInfos = {0}; int32_t dnodeInitMClient() { if (!dnodeReadMnodeIpList()) { - memset(&tsMpeerIpList, 0, sizeof(SRpcIpSet)); - memset(&tsMpeerInfos, 0, sizeof(SDMNodeInfos)); - tsMpeerIpList.port = tsMnodeDnodePort; - tsMpeerIpList.numOfIps = 1; - tsMpeerIpList.ip[0] = inet_addr(tsMasterIp); + memset(&tsMnodeIpList, 0, sizeof(SRpcIpSet)); + memset(&tsMnodeInfos, 0, sizeof(SDMNodeInfos)); + tsMnodeIpList.port = tsMnodeDnodePort; + tsMnodeIpList.numOfIps = 1; + tsMnodeIpList.ip[0] = inet_addr(tsMasterIp); if (tsSecondIp[0]) { - tsMpeerIpList.numOfIps = 2; - tsMpeerIpList.ip[1] = inet_addr(tsSecondIp); + tsMnodeIpList.numOfIps = 2; + tsMnodeIpList.ip[1] = inet_addr(tsSecondIp); } } else { - SRpcIpSet mgmtIpSet = {0}; - tsMpeerIpList.inUse = tsMpeerInfos.inUse; - tsMpeerIpList.numOfIps = tsMpeerInfos.nodeNum; - tsMpeerIpList.port = tsMpeerInfos.nodeInfos[0].nodePort; - for (int32_t i = 0; i < tsMpeerInfos.nodeNum; i++) { - tsMpeerIpList.ip[i] = tsMpeerInfos.nodeInfos[i].nodeIp; + tsMnodeIpList.inUse = tsMnodeInfos.inUse; + tsMnodeIpList.numOfIps = tsMnodeInfos.nodeNum; + tsMnodeIpList.port = tsMnodeInfos.nodeInfos[0].nodePort; + for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) { + tsMnodeIpList.ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp; } } @@ -123,17 +122,17 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { mgmtIpSet.ip[i] = htonl(mpeers->nodeInfos[i].nodeIp); } - if (memcmp(&mgmtIpSet, &tsMpeerIpList, sizeof(SRpcIpSet)) != 0) { - memcpy(&tsMpeerIpList, &mgmtIpSet, sizeof(SRpcIpSet)); - memcpy(&tsMpeerInfos, mpeers, sizeof(SDMNodeInfos)); - dPrint("mnode ip list is changed, numOfIps:%d inUse:%d", tsMpeerInfos.nodeNum, tsMpeerInfos.inUse); + if (memcmp(&mgmtIpSet, &tsMnodeIpList, sizeof(SRpcIpSet)) != 0) { + memcpy(&tsMnodeIpList, &mgmtIpSet, sizeof(SRpcIpSet)); + memcpy(&tsMnodeInfos, mpeers, sizeof(SDMNodeInfos)); + dPrint("mnode ip list is changed, numOfIps:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse); for (int32_t i = 0; i < mpeers->nodeNum; i++) { - tsMpeerInfos.nodeInfos[i].nodeId = htonl(mpeers->nodeInfos[i].nodeId); - tsMpeerInfos.nodeInfos[i].nodeIp = htonl(mpeers->nodeInfos[i].nodeIp); - tsMpeerInfos.nodeInfos[i].nodePort = htons(mpeers->nodeInfos[i].nodePort); - dPrint("mnode:%d, ip:%s:%u name:%s", tsMpeerInfos.nodeInfos[i].nodeId, - taosIpStr(tsMpeerInfos.nodeInfos[i].nodeId), tsMpeerInfos.nodeInfos[i].nodePort, - tsMpeerInfos.nodeInfos[i].nodeName); + tsMnodeInfos.nodeInfos[i].nodeId = htonl(mpeers->nodeInfos[i].nodeId); + tsMnodeInfos.nodeInfos[i].nodeIp = htonl(mpeers->nodeInfos[i].nodeIp); + tsMnodeInfos.nodeInfos[i].nodePort = htons(mpeers->nodeInfos[i].nodePort); + dPrint("mnode:%d, ip:%s:%u name:%s", tsMnodeInfos.nodeInfos[i].nodeId, + taosIpStr(tsMnodeInfos.nodeInfos[i].nodeId), tsMnodeInfos.nodeInfos[i].nodePort, + tsMnodeInfos.nodeInfos[i].nodeName); } dnodeSaveMnodeIpList(); } @@ -150,7 +149,7 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) { if (tsDnodeMClientRpc) { - rpcSendRequest(tsDnodeMClientRpc, &tsMpeerIpList, rpcMsg); + rpcSendRequest(tsDnodeMClientRpc, &tsMnodeIpList, rpcMsg); } } @@ -185,14 +184,14 @@ static bool dnodeReadMnodeIpList() { dError("failed to read mnode mgmtIpList.json, inUse not found"); goto PARSE_OVER; } - tsMpeerInfos.inUse = inUse->valueint; + tsMnodeInfos.inUse = inUse->valueint; cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum"); if (!nodeNum || nodeNum->type != cJSON_Number) { dError("failed to read mnode mgmtIpList.json, nodeNum not found"); goto PARSE_OVER; } - tsMpeerInfos.nodeNum = nodeNum->valueint; + tsMnodeInfos.nodeNum = nodeNum->valueint; cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { @@ -201,7 +200,7 @@ static bool dnodeReadMnodeIpList() { } int size = cJSON_GetArraySize(nodeInfos); - if (size != tsMpeerInfos.nodeNum) { + if (size != tsMnodeInfos.nodeNum) { dError("failed to read mnode mgmtIpList.json, nodeInfos size not matched"); goto PARSE_OVER; } @@ -215,37 +214,37 @@ static bool dnodeReadMnodeIpList() { dError("failed to read mnode mgmtIpList.json, nodeId not found"); goto PARSE_OVER; } - tsMpeerInfos.nodeInfos[i].nodeId = nodeId->valueint; + tsMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint; cJSON *nodeIp = cJSON_GetObjectItem(nodeInfo, "nodeIp"); if (!nodeIp || nodeIp->type != cJSON_String || nodeIp->valuestring == NULL) { dError("failed to read mnode mgmtIpList.json, nodeIp not found"); goto PARSE_OVER; } - tsMpeerInfos.nodeInfos[i].nodeIp = inet_addr(nodeIp->valuestring); + tsMnodeInfos.nodeInfos[i].nodeIp = inet_addr(nodeIp->valuestring); cJSON *nodePort = cJSON_GetObjectItem(nodeInfo, "nodePort"); if (!nodePort || nodePort->type != cJSON_Number) { dError("failed to read mnode mgmtIpList.json, nodePort not found"); goto PARSE_OVER; } - tsMpeerInfos.nodeInfos[i].nodePort = (uint16_t)nodePort->valueint; + tsMnodeInfos.nodeInfos[i].nodePort = (uint16_t)nodePort->valueint; cJSON *nodeName = cJSON_GetObjectItem(nodeInfo, "nodeName"); if (!nodeIp || nodeName->type != cJSON_String || nodeName->valuestring == NULL) { dError("failed to read mnode mgmtIpList.json, nodeName not found"); goto PARSE_OVER; } - strncpy(tsMpeerInfos.nodeInfos[i].nodeName, nodeName->valuestring, TSDB_NODE_NAME_LEN); + strncpy(tsMnodeInfos.nodeInfos[i].nodeName, nodeName->valuestring, TSDB_NODE_NAME_LEN); } ret = true; - dPrint("read mnode iplist successed, numOfIps:%d inUse:%d", tsMpeerInfos.nodeNum, tsMpeerInfos.inUse); - for (int32_t i = 0; i < tsMpeerInfos.nodeNum; i++) { - dPrint("mnode:%d, ip:%s:%u name:%s", tsMpeerInfos.nodeInfos[i].nodeId, - taosIpStr(tsMpeerInfos.nodeInfos[i].nodeId), tsMpeerInfos.nodeInfos[i].nodePort, - tsMpeerInfos.nodeInfos[i].nodeName); + dPrint("read mnode iplist successed, numOfIps:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse); + for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) { + dPrint("mnode:%d, ip:%s:%u name:%s", tsMnodeInfos.nodeInfos[i].nodeId, + taosIpStr(tsMnodeInfos.nodeInfos[i].nodeId), tsMnodeInfos.nodeInfos[i].nodePort, + tsMnodeInfos.nodeInfos[i].nodeName); } PARSE_OVER: @@ -265,15 +264,15 @@ static void dnodeSaveMnodeIpList() { char * content = calloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsMpeerInfos.inUse); - len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsMpeerInfos.nodeNum); + len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsMnodeInfos.inUse); + len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsMnodeInfos.nodeNum); len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); - for (int32_t i = 0; i < tsMpeerInfos.nodeNum; i++) { - len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsMpeerInfos.nodeInfos[i].nodeId); - len += snprintf(content + len, maxLen - len, " \"nodeIp\": \"%s\",\n", taosIpStr(tsMpeerInfos.nodeInfos[i].nodeIp)); - len += snprintf(content + len, maxLen - len, " \"nodePort\": %u,\n", tsMpeerInfos.nodeInfos[i].nodePort); - len += snprintf(content + len, maxLen - len, " \"nodeName\": \"%s\"\n", tsMpeerInfos.nodeInfos[i].nodeName); - if (i < tsMpeerInfos.nodeNum -1) { + for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) { + len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsMnodeInfos.nodeInfos[i].nodeId); + len += snprintf(content + len, maxLen - len, " \"nodeIp\": \"%s\",\n", taosIpStr(tsMnodeInfos.nodeInfos[i].nodeIp)); + len += snprintf(content + len, maxLen - len, " \"nodePort\": %u,\n", tsMnodeInfos.nodeInfos[i].nodePort); + len += snprintf(content + len, maxLen - len, " \"nodeName\": \"%s\"\n", tsMnodeInfos.nodeInfos[i].nodeName); + if (i < tsMnodeInfos.nodeNum -1) { len += snprintf(content + len, maxLen - len, " },{\n"); } else { len += snprintf(content + len, maxLen - len, " }]\n"); @@ -289,9 +288,9 @@ static void dnodeSaveMnodeIpList() { } uint32_t dnodeGetMnodeMasteIp() { - return tsMpeerIpList.ip[tsMpeerIpList.inUse]; + return tsMnodeIpList.ip[tsMnodeIpList.inUse]; } void* dnodeGetMpeerInfos() { - return &tsMpeerInfos; + return &tsMnodeInfos; } \ No newline at end of file diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt index b830695f52..2e975f089c 100644 --- a/src/mnode/CMakeLists.txt +++ b/src/mnode/CMakeLists.txt @@ -13,5 +13,5 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(mnode ${SRC}) - TARGET_LINK_LIBRARIES(mnode trpc tutil pthread) + #TARGET_LINK_LIBRARIES(mnode trpc tutil pthread) ENDIF () \ No newline at end of file diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index ecf6887b42..95f3e6d39b 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -20,6 +20,18 @@ extern "C" { #endif +typedef enum { + SDB_TABLE_MNODE = 0, + SDB_TABLE_DNODE = 1, + SDB_TABLE_ACCOUNT = 2, + SDB_TABLE_USER = 3, + SDB_TABLE_DB = 4, + SDB_TABLE_VGROUP = 5, + SDB_TABLE_STABLE = 6, + SDB_TABLE_CTABLE = 7, + SDB_TABLE_MAX = 8 +} ESdbTable; + typedef enum { SDB_KEY_TYPE_STRING, SDB_KEY_TYPE_AUTO @@ -34,8 +46,6 @@ typedef struct { ESdbOperType type; void * table; void * pObj; - int64_t version; - int32_t maxRowSize; int32_t rowSize; void * rowData; } SSdbOperDesc; @@ -45,6 +55,7 @@ typedef struct { int32_t hashSessions; int32_t maxRowSize; int32_t refCountPos; + ESdbTable tableId; ESdbKeyType keyType; int32_t (*insertFp)(SSdbOperDesc *pOper); int32_t (*deleteFp)(SSdbOperDesc *pOper); @@ -52,8 +63,12 @@ typedef struct { int32_t (*encodeFp)(SSdbOperDesc *pOper); int32_t (*decodeFp)(SSdbOperDesc *pDesc); int32_t (*destroyFp)(SSdbOperDesc *pDesc); + int32_t (*updateAllFp)(); } SSdbTableDesc; +int32_t sdbInit(); +void sdbCleanUp(); + void * sdbOpenTable(SSdbTableDesc *desc); void sdbCloseTable(void *handle); diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index b1931347a7..2de9656e3e 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -88,14 +88,9 @@ static int32_t mgmtDbActionUpdate(SSdbOperDesc *pOper) { static int32_t mgmtDbActionEncode(SSdbOperDesc *pOper) { SDbObj *pDb = pOper->pObj; - - if (pOper->maxRowSize < tsDbUpdateSize) { - return -1; - } else { - memcpy(pOper->rowData, pDb, tsDbUpdateSize); - pOper->rowSize = tsDbUpdateSize; - return TSDB_CODE_SUCCESS; - } + memcpy(pOper->rowData, pDb, tsDbUpdateSize); + pOper->rowSize = tsDbUpdateSize; + return TSDB_CODE_SUCCESS; } static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) { @@ -107,11 +102,16 @@ static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } +static int32_t mgmtDbActionUpdateAll() { + return 0; +} + int32_t mgmtInitDbs() { SDbObj tObj; tsDbUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { + .tableId = SDB_TABLE_DB, .tableName = "dbs", .hashSessions = TSDB_MAX_DBS, .maxRowSize = tsDbUpdateSize, @@ -123,6 +123,7 @@ int32_t mgmtInitDbs() { .encodeFp = mgmtDbActionEncode, .decodeFp = mgmtDbActionDecode, .destroyFp = mgmtDbActionDestroy, + .updateAllFp = mgmtDbActionUpdateAll }; tsDbSdb = sdbOpenTable(&tableDesc); @@ -136,7 +137,7 @@ int32_t mgmtInitDbs() { mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_DB, mgmtProcessDropDbMsg); mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_DB, mgmtGetDbMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_DB, mgmtRetrieveDbs); - + mTrace("db data is initialized"); return 0; } diff --git a/src/mnode/src/mgmtMain.c b/src/mnode/src/mgmtMain.c index cb3c9fae1e..a074060f52 100644 --- a/src/mnode/src/mgmtMain.c +++ b/src/mnode/src/mgmtMain.c @@ -105,7 +105,12 @@ int32_t mgmtStartSystem() { } if (mgmtInitTables() < 0) { - mError("failed to init meters"); + mError("failed to init tables"); + return -1; + } + + if (sdbInit() < 0) { + mError("failed to init sdb"); return -1; } @@ -158,6 +163,7 @@ void mgmtCleanUpSystem() { clusterCleanUp(); mgmtCleanUpUsers(); acctCleanUp(); + sdbCleanUp(); taosTmrCleanUp(tsMgmtTmr); mPrint("mgmt is cleaned up"); } diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index f3fc7d07f5..6a229e716c 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -22,37 +22,30 @@ #include "tlog.h" #include "trpc.h" #include "tutil.h" +#include "twal.h" +#include "tsync.h" #include "hashint.h" #include "hashstr.h" - #include "mgmtSdb.h" -#define abs(x) (((x) < 0) ? -(x) : (x)) -#define SDB_MAX_PEERS 4 -#define SDB_DELIMITER 0xFFF00F00 -#define SDB_ENDCOMMIT 0xAFFFAAAF - typedef struct { - uint64_t swVersion; - int16_t sdbFileVersion; - char reserved[2]; - TSCKSUM checkSum; -} SSdbHeader; + int32_t code; + int64_t version; + void * pSync; + void * pWal; + sem_t sem; + pthread_mutex_t mutex; +} SSdbSync; typedef struct _SSdbTable { - SSdbHeader header; - char tableName[TSDB_DB_NAME_LEN]; - char fileName[TSDB_FILENAME_LEN]; + char tableName[TSDB_DB_NAME_LEN + 1]; + ESdbTable tableId; ESdbKeyType keyType; - int32_t tableId; int32_t hashSessions; int32_t maxRowSize; int32_t refCountPos; int32_t autoIndex; - int32_t fd; int64_t numOfRows; - int64_t version; - int64_t fileSize; void * iHandle; int32_t (*insertFp)(SSdbOperDesc *pDesc); int32_t (*deleteFp)(SSdbOperDesc *pOper); @@ -60,55 +53,48 @@ typedef struct _SSdbTable { int32_t (*decodeFp)(SSdbOperDesc *pOper); int32_t (*encodeFp)(SSdbOperDesc *pOper); int32_t (*destroyFp)(SSdbOperDesc *pOper); + int32_t (*updateAllFp)(); pthread_mutex_t mutex; } SSdbTable; typedef struct { - int64_t version; - int64_t offset; int32_t rowSize; void * row; } SRowMeta; -typedef struct { - int32_t delimiter; - int32_t rowSize; - int64_t version; - char data[]; -} SRowHead; - typedef enum { - SDB_FORWARD_TYPE_INSERT, - SDB_FORWARD_TYPE_DELETE, - SDB_FORWARD_TYPE_UPDATE -} ESdbForwardType; + SDB_ACTION_INSERT, + SDB_ACTION_DELETE, + SDB_ACTION_UPDATE +} ESdbActionType; -typedef struct { - ESdbForwardType type; - int32_t tableId; - int64_t version; - int32_t rowSize; - void * rowData; -} SForwardMsg; - -extern char version[]; -const int16_t sdbFileVersion = 2; -int32_t (*mpeerForwardRequestFp)(SForwardMsg *forwardMsg) = NULL; - -static SSdbTable *sdbTableList[10] = {0}; -static int32_t sdbNumOfTables = 0; -static uint64_t sdbVersion = 0; +static SSdbTable *tsSdbTableList[SDB_TABLE_MAX] = {0}; +static int32_t tsSdbNumOfTables = 0; +static SSdbSync * tsSdbSync; static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash}; static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash}; -static void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash}; +static void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash}; static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData}; -static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash}; +static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash}; static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData}; +static int sdbProcessWrite(void *param, void *data, int type); -uint64_t sdbGetVersion() { return sdbVersion; } -int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->version; } -int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } +uint64_t sdbGetVersion() { return tsSdbSync->version; } +int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->autoIndex; } +int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } + +static char *sdbGetActionStr(int32_t action) { + switch (action) { + case SDB_ACTION_INSERT: + return "insert"; + case SDB_ACTION_DELETE: + return "delete"; + case SDB_ACTION_UPDATE: + return "update"; + } + return "invalid"; +} static char *sdbGetkeyStr(SSdbTable *pTable, void *row) { static char str[16]; @@ -123,300 +109,94 @@ static char *sdbGetkeyStr(SSdbTable *pTable, void *row) { } } -static int32_t sdbForwardDbReqToPeer(SForwardMsg *forwardMsg) { - if (mpeerForwardRequestFp) { - return mpeerForwardRequestFp(forwardMsg); - } else { - return 0; - } +static void *sdbGetTableFromId(int32_t tableId) { + return tsSdbTableList[tableId]; } -static void sdbFinishCommit(SSdbTable *pTable) { - uint32_t sdbEcommit = SDB_ENDCOMMIT; - off_t offset = lseek(pTable->fd, 0, SEEK_END); - assert(offset == pTable->fileSize); - twrite(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); - pTable->fileSize += sizeof(sdbEcommit); -} +// static void mpeerConfirmForward(void *ahandle, void *param, int32_t code) { +// sem_post(&tsSdbSync->sem); +// mPrint("mpeerConfirmForward"); +// } -static int32_t sdbOpenSdbFile(SSdbTable *pTable) { - struct stat fstat, ofstat; - uint64_t size; - char * dirc = NULL; - char * basec = NULL; - union { - char cversion[64]; - uint64_t iversion; - } swVersion; - - memcpy(swVersion.cversion, version, sizeof(uint64_t)); - - // check sdb.db and .sdb.db status - char fn[TSDB_FILENAME_LEN] = "\0"; - dirc = strdup(pTable->fileName); - basec = strdup(pTable->fileName); - sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); - tfree(dirc); - tfree(basec); - if (stat(fn, &ofstat) == 0) { // .sdb.db file exists - if (stat(pTable->fileName, &fstat) == 0) { - remove(fn); - } else { - remove(pTable->fileName); - rename(fn, pTable->fileName); - } - } - - pTable->fd = open(pTable->fileName, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - if (pTable->fd < 0) { - sdbError("table:%s, failed to open file:%s", pTable->tableName, pTable->fileName); - return -1; - } - - pTable->fileSize = 0; - stat(pTable->fileName, &fstat); - size = sizeof(pTable->header); - - if (fstat.st_size == 0) { - pTable->header.swVersion = swVersion.iversion; - pTable->header.sdbFileVersion = sdbFileVersion; - if (taosCalcChecksumAppend(0, (uint8_t *)(&pTable->header), size) < 0) { - sdbError("table:%s, failed to get file header checksum, file:%s", pTable->tableName, pTable->fileName); - tclose(pTable->fd); - return -1; - } - twrite(pTable->fd, &(pTable->header), size); - pTable->fileSize += size; - sdbFinishCommit(pTable); - } else { - uint32_t sdbEcommit = 0; - off_t offset = lseek(pTable->fd, -(sizeof(sdbEcommit)), SEEK_END); - while (offset > 0) { - read(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); - if (sdbEcommit == SDB_ENDCOMMIT) { - ftruncate(pTable->fd, offset + sizeof(sdbEcommit)); - break; - } - offset = lseek(pTable->fd, -(sizeof(sdbEcommit) + 1), SEEK_CUR); - } - lseek(pTable->fd, 0, SEEK_SET); - - ssize_t tsize = read(pTable->fd, &(pTable->header), size); - if (tsize < size) { - sdbError("table:%s, failed to read sdb file header, file:%s", pTable->tableName, pTable->fileName); - tclose(pTable->fd); - return -1; - } - - if (pTable->header.swVersion != swVersion.iversion) { - sdbWarn("table:%s, sdb file:%s version not match software version", pTable->tableName, pTable->fileName); - } - - if (!taosCheckChecksumWhole((uint8_t *)(&pTable->header), size)) { - sdbError("table:%s, sdb file header is broken since checksum mismatch, file:%s", pTable->tableName, pTable->fileName); - tclose(pTable->fd); - return -1; - } - - pTable->fileSize += size; - // skip end commit symbol - lseek(pTable->fd, sizeof(sdbEcommit), SEEK_CUR); - pTable->fileSize += sizeof(sdbEcommit); - } - - pTable->numOfRows = 0; - - return pTable->fd; -} - -static int32_t sdbInitTableByFile(SSdbTable *pTable) { - sdbTrace("table:%s, open sdb file:%s for read", pTable->tableName, pTable->fileName); - if (sdbOpenSdbFile(pTable) < 0) { - sdbError("table:%s, failed to open sdb file:%s for read", pTable->tableName, pTable->fileName); - return -1; - } +static int32_t sdbForwardDbReqToPeer(SWalHead *pHead) { + // int32_t code = syncForwardToPeer(NULL, pHead, NULL); + // if (code < 0) { + // return code; + // } - int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("table:%s, failed to allocate row head memory, sdb:%s", pTable->tableName, pTable->tableName); + // sem_wait(&tsSdbSync->sem); + // return tsSdbSync->code; + return TSDB_CODE_SUCCESS; +} + +int32_t sdbInit() { + tsSdbSync = calloc(1, sizeof(SSdbSync)); + sem_init(&tsSdbSync->sem, 0, 0); + pthread_mutex_init(&tsSdbSync->mutex, NULL); + + SWalCfg walCfg = {.commitLog = 2, .wals = 2, .keep = 1}; + tsSdbSync->pWal = walOpen(tsMnodeDir, &walCfg); + if (tsSdbSync->pWal == NULL) { + sdbError("failed to open sdb in %s", tsMnodeDir); return -1; } - int32_t numOfChanged = 0; - int32_t maxAutoIndex = 0; - while (1) { - memset(rowHead, 0, total_size); + sdbTrace("open sdb file for read"); + walRestore(tsSdbSync->pWal, tsSdbSync, sdbProcessWrite); - int32_t bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); - if (bytes < 0) { - sdbError("table:%s, failed to read sdb file:%s", pTable->tableName, pTable->fileName); - tfree(rowHead); - return -1; + int32_t totalRows = 0; + int32_t numOfTables = 0; + for (int32_t tableId = SDB_TABLE_DNODE; tableId < SDB_TABLE_MAX; ++tableId) { + SSdbTable *pTable = sdbGetTableFromId(tableId); + if (pTable == NULL) continue; + if (pTable->updateAllFp) { + (*pTable->updateAllFp)(); } - if (bytes == 0) break; - - if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) { - pTable->fileSize++; - lseek(pTable->fd, -(bytes - 1), SEEK_CUR); - continue; - } - - if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) { - sdbError("table:%s, error row size in sdb filesize:%d, version:%d rowSize:%d maxRowSize:%d", pTable->tableName, - pTable->fileSize, rowHead->version, rowHead->rowSize, pTable->maxRowSize); - pTable->fileSize += sizeof(SRowHead); - continue; - } - - bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); - if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { - // TODO: Here may cause pTable->fileSize not end of the file - sdbError("table:%s, failed to read sdb file, version:%d rowSize:%d", pTable->tableName, rowHead->version, - rowHead->rowSize); - break; - } - - int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) { - sdbError("table:%s, error sdb checksum, version:%d, skip", pTable->tableName, rowHead->version); - pTable->fileSize += real_size; - continue; - } - - if (pTable->keyType == SDB_KEY_TYPE_AUTO) { - maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data); - } - - pTable->version = MAX(pTable->version, abs(rowHead->version)); - - void *pMetaRow = sdbGetRow(pTable, rowHead->data); - if (pMetaRow == NULL) { - if (rowHead->version < 0) { - sdbError("table:%s, error sdb negative version:%d, record:%s, skip", pTable->tableName, rowHead->version, - sdbGetkeyStr(pTable, rowHead->data)); - } else { - SRowMeta rowMeta; - rowMeta.version = rowHead->version; - rowMeta.offset = pTable->fileSize; - rowMeta.rowSize = rowHead->rowSize; - SSdbOperDesc oper = { - .table = pTable, - .rowData = rowHead->data, - .rowSize = rowHead->rowSize - }; - int32_t code = (*pTable->decodeFp)(&oper); - if (code == TSDB_CODE_SUCCESS) { - rowMeta.row = oper.pObj; - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - pTable->numOfRows++; - sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read new record:%s", - pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); - } else { - sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s", - pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); - } - } - } else { - if (rowHead->version < 0) { - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - pTable->numOfRows--; - sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read deleted record:%s", - pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); - } else { - SRowMeta rowMeta; - rowMeta.version = rowHead->version; - rowMeta.offset = pTable->fileSize; - rowMeta.rowSize = rowHead->rowSize; - SSdbOperDesc oper = { - .table = pTable, - .rowData = rowHead->data, - .rowSize = rowHead->rowSize, - .pObj = pMetaRow - }; - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - - int32_t code = (*pTable->decodeFp)(&oper); - if (code == TSDB_CODE_SUCCESS) { - rowMeta.row = oper.pObj; - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read updated record:%s", - pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); - } else { - sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s", - pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); - } - } - numOfChanged++; - } - - pTable->fileSize += real_size; - pTable->fileSize += 4; - lseek(pTable->fd, 4, SEEK_CUR); + totalRows += pTable->numOfRows; + numOfTables++; + sdbTrace("table:%s, is initialized, numOfRows:%d", pTable->tableName, pTable->numOfRows); } - void *pNode = NULL; - while (1) { - SRowMeta * pMeta; - pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); - if (pMeta == NULL) break; - - SSdbOperDesc oper = { - .pObj = pMeta->row, - .table = pTable, - .version = pMeta->version, - }; - - sdbIncRef(pTable, oper.pObj); - int32_t code = (*pTable->insertFp)(&oper); - if (code != TSDB_CODE_SUCCESS) { - sdbError("table:%s, failed to insert record:%s", pTable->tableName, sdbGetkeyStr(pTable, rowHead->data)); - } - } - - sdbVersion += pTable->version; - - if (pTable->keyType == SDB_KEY_TYPE_AUTO) { - pTable->autoIndex = maxAutoIndex; - } - - tfree(rowHead); - return 0; + sdbTrace("sdb is initialized, version:%d totalRows:%d numOfTables:%d", tsSdbSync->version, totalRows, numOfTables); + return TSDB_CODE_SUCCESS; } -void *sdbOpenTable(SSdbTableDesc *pDesc) { - SSdbTable *pTable = (SSdbTable *)calloc(1, sizeof(SSdbTable)); - if (pTable == NULL) return NULL; - - pTable->keyType = pDesc->keyType; - pTable->hashSessions = pDesc->hashSessions; - pTable->maxRowSize = pDesc->maxRowSize; - pTable->refCountPos = pDesc->refCountPos; - pTable->insertFp = pDesc->insertFp; - pTable->deleteFp = pDesc->deleteFp; - pTable->updateFp = pDesc->updateFp; - pTable->encodeFp = pDesc->encodeFp; - pTable->decodeFp = pDesc->decodeFp; - pTable->destroyFp = pDesc->destroyFp; - strcpy(pTable->tableName, pDesc->tableName); - sprintf(pTable->fileName, "%s/%s.db", tsMnodeDir, pTable->tableName); - - if (sdbInitIndexFp[pTable->keyType] != NULL) { - pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SRowMeta)); +void sdbCleanUp() { + if (tsSdbSync) { + sem_destroy(&tsSdbSync->sem); + pthread_mutex_destroy(&tsSdbSync->mutex); + walClose(tsSdbSync->pWal); + tsSdbSync = NULL; } +} - pthread_mutex_init(&pTable->mutex, NULL); +void sdbIncRef(void *handle, void *pRow) { + if (pRow) { + SSdbTable *pTable = handle; + int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos); + atomic_add_fetch_32(pRefCount, 1); + //if (0 && strcmp(pTable->tableName, "dnodes") == 0) { + sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); + //} + } +} - if (sdbInitTableByFile(pTable) < 0) return NULL; - - pTable->tableId = sdbNumOfTables++; - sdbTableList[pTable->tableId] = pTable; - - sdbTrace("table:%s, is initialized, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d numOfTables:%d", - pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbNumOfTables); - - return pTable; +void sdbDecRef(void *handle, void *pRow) { + if (pRow) { + SSdbTable *pTable = handle; + int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos); + int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); + //if (0 && strcmp(pTable->tableName, "dnodes") == 0) { + sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); + //} + int8_t* updateEnd = pRow + pTable->refCountPos - 1; + if (refCount <= 0 && *updateEnd) { + sdbTrace("table:%s, record:%s:%s:%d is destroyed", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); + SSdbOperDesc oper = {.pObj = pRow}; + (*pTable->destroyFp)(&oper); + } + } } static SRowMeta *sdbGetRowMeta(void *handle, void *key) { @@ -430,34 +210,6 @@ static SRowMeta *sdbGetRowMeta(void *handle, void *key) { return pMeta; } -void sdbIncRef(void *handle, void *pRow) { - if (pRow) { - SSdbTable *pTable = handle; - int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos); - atomic_add_fetch_32(pRefCount, 1); - if (0 && strcmp(pTable->tableName, "dnodes") == 0) { - sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); - } - } -} - -void sdbDecRef(void *handle, void *pRow) { - if (pRow) { - SSdbTable *pTable = handle; - int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos); - int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); - if (0 && strcmp(pTable->tableName, "dnodes") == 0) { - sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); - } - int8_t* updateEnd = pRow + pTable->refCountPos - 1; - if (refCount <= 0 && *updateEnd) { - sdbTrace("table:%s, record:%s:%s:%d is destroyed", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); - SSdbOperDesc oper = {.pObj = pRow}; - (*pTable->destroyFp)(&oper); - } - } -} - void *sdbGetRow(void *handle, void *key) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta; @@ -476,12 +228,167 @@ void *sdbGetRow(void *handle, void *key) { return pMeta->row; } +static int32_t sdbInsertLocal(SSdbTable* pTable, SSdbOperDesc *pOper) { + SRowMeta rowMeta; + rowMeta.rowSize = pOper->rowSize; + rowMeta.row = pOper->pObj; + + pthread_mutex_lock(&pTable->mutex); + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj, &rowMeta); + sdbIncRef(pTable, pOper->pObj); + pTable->numOfRows++; + pthread_mutex_unlock(&pTable->mutex); + + sdbTrace("table:%s, insert record:%s, numOfRows:%d", pTable->tableName, + sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows); + + (*pTable->insertFp)(pOper); + return TSDB_CODE_SUCCESS; +} + +static int32_t sdbDeleteLocal(SSdbTable* pTable, SSdbOperDesc *pOper) { + pthread_mutex_lock(&pTable->mutex); + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj); + pTable->numOfRows--; + pthread_mutex_unlock(&pTable->mutex); + + sdbTrace("table:%s, delete record:%s, numOfRows:%d", pTable->tableName, + sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows); + + (*pTable->deleteFp)(pOper); + int8_t* updateEnd = pOper->pObj + pTable->refCountPos - 1; + *updateEnd = 1; + sdbDecRef(pTable, pOper->pObj); + + return TSDB_CODE_SUCCESS; +} + +static int32_t sdbUpdateLocal(SSdbTable* pTable, SSdbOperDesc *pOper) { + sdbTrace("table:%s, update record:%s, numOfRows:%d", pTable->tableName, + sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows); + + (*pTable->updateFp)(pOper); + return TSDB_CODE_SUCCESS; +} + +static int sdbProcessWrite(void *param, void *data, int type) { + SWalHead *pHead = data; + int32_t code = 0; + int32_t tableId = pHead->msgType / 10; + int32_t action = pHead->msgType % 10; + + SSdbTable *pTable = sdbGetTableFromId(tableId); + assert(pTable != NULL); + + if (pHead->version == 0) { + // from mgmt, update version + pthread_mutex_lock(&tsSdbSync->mutex); + tsSdbSync->version++; + pHead->version = tsSdbSync->version; + + code = sdbForwardDbReqToPeer(pHead); + if (code != TSDB_CODE_SUCCESS) { + pthread_mutex_unlock(&tsSdbSync->mutex); + sdbError("table:%s, failed to forward %s record:%s from file, version:%" PRId64 ", reason:%s", pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code)); + return code; + } + + code = walWrite(tsSdbSync->pWal, pHead); + pthread_mutex_unlock(&tsSdbSync->mutex); + + if (code < 0) { + sdbError("table:%s, failed to %s record:%s to file, version:%" PRId64 ", reason:%s", pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code)); + } else { + sdbTrace("table:%s, success to %s record:%s to file, version:%" PRId64, pTable->tableName, sdbGetActionStr(action), + sdbGetkeyStr(pTable, pHead->cont), pHead->version); + } + + walFsync(tsSdbSync->pWal); + free(pHead); + + return code; + } else { + // for data from WAL or forward, version may be smaller + pthread_mutex_lock(&tsSdbSync->mutex); + + if (pHead->version <= tsSdbSync->version) { + pthread_mutex_unlock(&tsSdbSync->mutex); + return TSDB_CODE_SUCCESS; + } else if (pHead->version != tsSdbSync->version + 1) { + pthread_mutex_unlock(&tsSdbSync->mutex); + sdbError("table:%s, failed to restore %s record:%s from file, version:%" PRId64 " too large, sdb version:%" PRId64, + pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, + tsSdbSync->version); + return TSDB_CODE_OTHERS; + } else { + tsSdbSync->version = pHead->version; + sdbTrace("table:%s, success to restore %s record:%s from file, version:%" PRId64, pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); + } + + + + code = -1; + if (action == SDB_ACTION_INSERT) { + SSdbOperDesc oper = { + .rowSize = pHead->len, + .rowData = pHead->cont, + .table = pTable, + }; + code = (*pTable->decodeFp)(&oper); + if (code < 0) { + sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); + pthread_mutex_unlock(&tsSdbSync->mutex); + return code; + } + + code = sdbInsertLocal(pTable, &oper); + } else if (action == SDB_ACTION_DELETE) { + SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont); + assert(rowMeta != NULL && rowMeta->row != NULL); + + SSdbOperDesc oper = { + .table = pTable, + .pObj = rowMeta->row, + }; + + code = sdbDeleteLocal(pTable, &oper); + } else if (action == SDB_ACTION_UPDATE) { + SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont); + assert(rowMeta != NULL && rowMeta->row != NULL); + + SSdbOperDesc oper1 = { + .table = pTable, + .pObj = rowMeta->row, + }; + sdbDeleteLocal(pTable, &oper1); + + SSdbOperDesc oper2 = { + .rowSize = pHead->len, + .rowData = pHead->cont, + .table = pTable, + }; + code = (*pTable->decodeFp)(&oper2); + if (code < 0) { + sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); + pthread_mutex_unlock(&tsSdbSync->mutex); + return code; + } + code = sdbInsertLocal(pTable, &oper2); + } + + pthread_mutex_unlock(&tsSdbSync->mutex); + return code; + } +} + int32_t sdbInsertRow(SSdbOperDesc *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; - if (pTable == NULL) { - sdbError("sdb tables is null"); - return TSDB_CODE_OTHERS; - } + if (pTable == NULL) return -1; if (sdbGetRow(pTable, pOper->pObj)) { sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj)); @@ -489,87 +396,33 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { return TSDB_CODE_ALREADY_THERE; } - pOper->maxRowSize = pTable->maxRowSize; - pthread_mutex_lock(&pTable->mutex); - - if (pOper->type == SDB_OPER_TYPE_GLOBAL) { - SForwardMsg forward = { - .type = SDB_FORWARD_TYPE_INSERT, - .tableId = pTable->tableId, - .version = pTable->version + 1, - .rowSize = pOper->rowSize, - .rowData = pOper->rowData, - }; - - if (sdbForwardDbReqToPeer(&forward) != 0) { - sdbError("table:%s, failed to forward record:%s version:%" PRId64 " sdbversion:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion); - pthread_mutex_unlock(&pTable->mutex); - return TSDB_CODE_OTHERS; - } - } - - int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)calloc(1, total_size); - if (rowHead == NULL) { - pthread_mutex_unlock(&pTable->mutex); - sdbError("table:%s, failed to allocate row head memory for record:%s version:%" PRId64 " sdbversion:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion); - return -1; - } - if (pTable->keyType == SDB_KEY_TYPE_AUTO) { + pthread_mutex_lock(&pTable->mutex); *((uint32_t *)pOper->pObj) = ++pTable->autoIndex; // let vgId increase from 2 if (pTable->autoIndex == 1 && strcmp(pTable->tableName, "vgroups") == 0) { *((uint32_t *)pOper->pObj) = ++pTable->autoIndex; } - } - pTable->version++; - sdbVersion++; - - pOper->rowData = rowHead->data; - (*pTable->encodeFp)(pOper); - rowHead->rowSize = pOper->rowSize; - rowHead->delimiter = SDB_DELIMITER; - rowHead->version = pTable->version; - assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); - - int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("table:%s, failed to get checksum while inserting", pTable->tableName); - pTable->version--; - sdbVersion--; pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; } - twrite(pTable->fd, rowHead, real_size); - pTable->fileSize += real_size; - sdbFinishCommit(pTable); - tfree(rowHead); - - // update in SDB layer - SRowMeta rowMeta; - rowMeta.version = pTable->version; - rowMeta.offset = pTable->fileSize; - rowMeta.rowSize = pOper->rowSize; - rowMeta.row = pOper->pObj; - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj, &rowMeta); - sdbIncRef(pTable, pOper->pObj); + if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + int32_t size = sizeof(SWalHead) + pTable->maxRowSize; + SWalHead *pHead = calloc(1, size); + pHead->version = 0; + pHead->len = pOper->rowSize; + pHead->msgType = pTable->tableId * 10 + SDB_ACTION_INSERT; - pTable->numOfRows++; + pOper->rowData = pHead->cont; + (*pTable->encodeFp)(pOper); + pHead->len = pOper->rowSize; + + int32_t code = sdbProcessWrite(tsSdbSync, pHead, pHead->msgType); + if (code < 0) return code; + } - pthread_mutex_unlock(&pTable->mutex); - - sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, insert record:%s, rowSize:%d fileSize:%" PRId64, - pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); - - (*pTable->insertFp)(pOper); - - return 0; + return sdbInsertLocal(pTable, pOper); } // row here can be object or null-terminated string @@ -586,204 +439,61 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { void * pMetaRow = pMeta->row; assert(pMetaRow != NULL); - pthread_mutex_lock(&pTable->mutex); - if (pOper->type == SDB_OPER_TYPE_GLOBAL) { - SForwardMsg forward = { - .type = SDB_FORWARD_TYPE_DELETE, - .tableId = pTable->tableId, - .version = pTable->version + 1, - .rowSize = pMeta->rowSize, - .rowData = pMeta->row, - }; + int32_t rowSize = 0; + switch (pTable->keyType) { + case SDB_KEY_TYPE_STRING: + rowSize = strlen((char *)pOper->pObj) + 1; + break; + case SDB_KEY_TYPE_AUTO: + rowSize = sizeof(uint64_t); + break; + default: + return -1; + } - if (sdbForwardDbReqToPeer(&forward) != 0) { - sdbError("table:%s, failed to delete record", pTable->tableName); - pthread_mutex_unlock(&pTable->mutex); - return -1; - } - } + int32_t size = sizeof(SWalHead) + rowSize; + SWalHead *pHead = calloc(1, size); + pHead->version = 0; + pHead->len = rowSize; + pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE; + memcpy(pHead->cont, pOper->pObj, rowSize); - int32_t total_size = sizeof(SRowHead) + pMeta->rowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)calloc(1, total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->tableName); - pthread_mutex_unlock(&pTable->mutex); - return -1; - } - - pTable->version++; - sdbVersion++; - - int32_t rowSize = 0; - switch (pTable->keyType) { - case SDB_KEY_TYPE_STRING: - rowSize = strlen((char *)pOper->pObj) + 1; - break; - case SDB_KEY_TYPE_AUTO: - rowSize = sizeof(uint64_t); - break; - default: - return -1; - } - - rowHead->delimiter = SDB_DELIMITER; - rowHead->rowSize = rowSize; - rowHead->version = -(pTable->version); - memcpy(rowHead->data, pOper->pObj, rowSize); - int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum while inserting, sdb:%s", pTable->tableName); - pTable->version--; - sdbVersion--; - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - - twrite(pTable->fd, rowHead, real_size); - pTable->fileSize += real_size; - sdbFinishCommit(pTable); - - tfree(rowHead); - - - sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, delete record:%s, rowSize:%d fileSize:%" PRId64, - pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); - - // Delete from current layer - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj); - - pTable->numOfRows--; - - pthread_mutex_unlock(&pTable->mutex); - - (*pTable->deleteFp)(pOper); - int8_t* updateEnd = pOper->pObj + pTable->refCountPos - 1; - *updateEnd = 1; - sdbDecRef(pTable, pOper->pObj); - return 0; + int32_t code = sdbProcessWrite(tsSdbSync, pHead, pHead->msgType); + if (code < 0) return code; + } + + return sdbDeleteLocal(pTable, pOper); } -// row here can be the object or the string info (encoded string) int32_t sdbUpdateRow(SSdbOperDesc *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; if (pTable == NULL) return -1; SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj); if (pMeta == NULL) { - sdbError("table:%s, failed to update record:%s, record is not there, sdbversion:%" PRId64 " version:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version); + sdbTrace("table:%s, record is not there, delete failed", pTable->tableName); return -1; } - void *pMetaRow = pMeta->row; + void * pMetaRow = pMeta->row; assert(pMetaRow != NULL); - pthread_mutex_lock(&pTable->mutex); - if (pOper->type == SDB_OPER_TYPE_GLOBAL) { - SForwardMsg forward = { - .type = SDB_FORWARD_TYPE_UPDATE, - .tableId = pTable->tableId, - .version = pTable->version + 1, - .rowSize = pOper->rowSize, - .rowData = pOper->rowData, - }; - if (sdbForwardDbReqToPeer(&forward) != 0) { - sdbError("table:%s, failed to update record", pTable->tableName); - pthread_mutex_unlock(&pTable->mutex); - return -1; - } - } + int32_t size = sizeof(SWalHead) + pTable->maxRowSize; + SWalHead *pHead = calloc(1, size); + pHead->version = 0; + pHead->msgType = pTable->tableId * 10 + SDB_ACTION_UPDATE; - int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)calloc(1, total_size); - if (rowHead == NULL) { - pthread_mutex_unlock(&pTable->mutex); - sdbError("table:%s, failed to allocate row head memory", pTable->tableName); - return -1; + pOper->rowData = pHead->cont; + (*pTable->encodeFp)(pOper); + pHead->len = pOper->rowSize; + + int32_t code = sdbProcessWrite(tsSdbSync, pHead, pHead->msgType); + if (code < 0) return code; } - if (pMetaRow != pOper->pObj) { - memcpy(rowHead->data, pOper->rowData, pOper->rowSize); - rowHead->rowSize = pOper->rowSize; - } else { - SSdbOperDesc oper = { - .table = pTable, - .rowData = rowHead->data, - .maxRowSize = pTable->maxRowSize, - .pObj = pOper->pObj - }; - (*pTable->encodeFp)(&oper); - rowHead->rowSize = oper.rowSize; - } - - pTable->version++; - sdbVersion++; - - int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - rowHead->delimiter = SDB_DELIMITER; - rowHead->version = pTable->version; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("table:%s, failed to get checksum, version:%d", pTable->tableName, rowHead->version); - pTable->version--; - sdbVersion--; - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - - twrite(pTable->fd, rowHead, real_size); - pTable->fileSize += real_size; - sdbFinishCommit(pTable); - - sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, update record:%s, rowSize:%d fileSize:%" PRId64, - pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); - - pMeta->version = pTable->version; - pMeta->offset = pTable->fileSize; - pMeta->rowSize = rowHead->rowSize; - - pthread_mutex_unlock(&pTable->mutex); - - (*pTable->updateFp)(pOper); // update in upper layer - - tfree(rowHead); - - return 0; -} - -void sdbCloseTable(void *handle) { - SSdbTable *pTable = (SSdbTable *)handle; - void * pNode = NULL; - - if (pTable == NULL) return; - - while (1) { - SRowMeta * pMeta; - pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); - if (pMeta == NULL) break; - - SSdbOperDesc oper = { - .pObj = pMeta->row, - .table = pTable, - .version = pMeta->version, - }; - - (*pTable->destroyFp)(&oper); - } - - if (sdbCleanUpIndexFp[pTable->keyType]) (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle); - - if (pTable->fd) tclose(pTable->fd); - - pthread_mutex_destroy(&pTable->mutex); - - sdbNumOfTables--; - sdbTrace("table:%s, is closed, version:%" PRId64 " numOfTables:%d", pTable->tableName, pTable->version, sdbNumOfTables); - - tfree(pTable); + return sdbUpdateLocal(pTable, pOper); } void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { @@ -801,3 +511,63 @@ void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { return pNode; } + +void *sdbOpenTable(SSdbTableDesc *pDesc) { + SSdbTable *pTable = (SSdbTable *)calloc(1, sizeof(SSdbTable)); + if (pTable == NULL) return NULL; + + strcpy(pTable->tableName, pDesc->tableName); + pTable->keyType = pDesc->keyType; + pTable->tableId = pDesc->tableId; + pTable->hashSessions = pDesc->hashSessions; + pTable->maxRowSize = pDesc->maxRowSize; + pTable->refCountPos = pDesc->refCountPos; + pTable->insertFp = pDesc->insertFp; + pTable->deleteFp = pDesc->deleteFp; + pTable->updateFp = pDesc->updateFp; + pTable->encodeFp = pDesc->encodeFp; + pTable->decodeFp = pDesc->decodeFp; + pTable->destroyFp = pDesc->destroyFp; + pTable->updateAllFp = pDesc->updateAllFp; + + if (sdbInitIndexFp[pTable->keyType] != NULL) { + pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SRowMeta)); + } + + pthread_mutex_init(&pTable->mutex, NULL); + + tsSdbNumOfTables++; + tsSdbTableList[pTable->tableId] = pTable; + return pTable; +} + +void sdbCloseTable(void *handle) { + SSdbTable *pTable = (SSdbTable *)handle; + if (pTable == NULL) return; + + tsSdbNumOfTables--; + tsSdbTableList[pTable->tableId] = NULL; + + void *pNode = NULL; + while (1) { + SRowMeta *pMeta; + pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); + if (pMeta == NULL) break; + + SSdbOperDesc oper = { + .pObj = pMeta->row, + .table = pTable, + }; + + (*pTable->destroyFp)(&oper); + } + + if (sdbCleanUpIndexFp[pTable->keyType]) { + (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle); + } + + pthread_mutex_destroy(&pTable->mutex); + + sdbTrace("table:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbNumOfTables); + tfree(pTable); +} diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 8dfe835f32..19b747b745 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -174,6 +174,7 @@ static int32_t mgmtChildTableActionUpdate(SSdbOperDesc *pOper) { } static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) { + const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS; SChildTableObj *pTable = pOper->pObj; assert(pTable != NULL && pOper->rowData != NULL); @@ -182,7 +183,7 @@ static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) { pOper->rowSize = tsChildTableUpdateSize; } else { int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - if (pOper->maxRowSize < tsChildTableUpdateSize + schemaSize) { + if (maxRowSize < tsChildTableUpdateSize + schemaSize) { return TSDB_CODE_INVALID_MSG_LEN; } memcpy(pOper->rowData, pTable, tsChildTableUpdateSize); @@ -224,35 +225,11 @@ static int32_t mgmtChildTableActionDecode(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtInitChildTables() { +static int32_t mgmtChildTableActionUpdateAll() { void *pNode = NULL; void *pLastNode = NULL; SChildTableObj *pTable = NULL; - SChildTableObj tObj; - tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; - - SSdbTableDesc tableDesc = { - .tableName = "ctables", - .hashSessions = tsMaxTables, - .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, - .keyType = SDB_KEY_TYPE_STRING, - .insertFp = mgmtChildTableActionInsert, - .deleteFp = mgmtChildTableActionDelete, - .updateFp = mgmtChildTableActionUpdate, - .encodeFp = mgmtChildTableActionEncode, - .decodeFp = mgmtChildTableActionDecode, - .destroyFp = mgmtChildTableActionDestroy, - }; - - tsChildTableSdb = sdbOpenTable(&tableDesc); - if (tsChildTableSdb == NULL) { - mError("failed to init child table data"); - return -1; - } - - pNode = NULL; while (1) { pLastNode = pNode; mgmtDecTableRef(pTable); @@ -328,6 +305,35 @@ static int32_t mgmtInitChildTables() { } } + return 0; +} + +static int32_t mgmtInitChildTables() { + SChildTableObj tObj; + tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableId = SDB_TABLE_CTABLE, + .tableName = "ctables", + .hashSessions = tsMaxTables, + .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, + .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .keyType = SDB_KEY_TYPE_STRING, + .insertFp = mgmtChildTableActionInsert, + .deleteFp = mgmtChildTableActionDelete, + .updateFp = mgmtChildTableActionUpdate, + .encodeFp = mgmtChildTableActionEncode, + .decodeFp = mgmtChildTableActionDecode, + .destroyFp = mgmtChildTableActionDestroy, + .updateAllFp = mgmtChildTableActionUpdateAll + }; + + tsChildTableSdb = sdbOpenTable(&tableDesc); + if (tsChildTableSdb == NULL) { + mError("failed to init child table data"); + return -1; + } + mTrace("child table is initialized"); return 0; } @@ -374,12 +380,14 @@ static int32_t mgmtSuperTableActionUpdate(SSdbOperDesc *pOper) { } static int32_t mgmtSuperTableActionEncode(SSdbOperDesc *pOper) { + const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS; + SSuperTableObj *pStable = pOper->pObj; assert(pOper->pObj != NULL && pOper->rowData != NULL); int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); - if (pOper->maxRowSize < tsSuperTableUpdateSize + schemaSize) { + if (maxRowSize < tsSuperTableUpdateSize + schemaSize) { return TSDB_CODE_INVALID_MSG_LEN; } @@ -411,11 +419,16 @@ static int32_t mgmtSuperTableActionDecode(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } +static int32_t mgmtSuperTableActionUpdateAll() { + return 0; +} + static int32_t mgmtInitSuperTables() { SSuperTableObj tObj; tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { + .tableId = SDB_TABLE_STABLE, .tableName = "stables", .hashSessions = TSDB_MAX_SUPER_TABLES, .maxRowSize = tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, @@ -427,6 +440,7 @@ static int32_t mgmtInitSuperTables() { .encodeFp = mgmtSuperTableActionEncode, .decodeFp = mgmtSuperTableActionDecode, .destroyFp = mgmtSuperTableActionDestroy, + .updateAllFp = mgmtSuperTableActionUpdateAll }; tsSuperTableSdb = sdbOpenTable(&tableDesc); diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 8e969b3c9c..b216e8f36d 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -70,14 +70,9 @@ static int32_t mgmtUserActionUpdate(SSdbOperDesc *pOper) { static int32_t mgmtUserActionEncode(SSdbOperDesc *pOper) { SUserObj *pUser = pOper->pObj; - - if (pOper->maxRowSize < tsUserUpdateSize) { - return -1; - } else { - memcpy(pOper->rowData, pUser, tsUserUpdateSize); - pOper->rowSize = tsUserUpdateSize; - return TSDB_CODE_SUCCESS; - } + memcpy(pOper->rowData, pUser, tsUserUpdateSize); + pOper->rowSize = tsUserUpdateSize; + return TSDB_CODE_SUCCESS; } static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) { @@ -89,11 +84,22 @@ static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } +static int32_t mgmtUserActionUpdateAll() { + SAcctObj *pAcct = acctGetAcct("root"); + mgmtCreateUser(pAcct, "root", "taosdata"); + mgmtCreateUser(pAcct, "monitor", tsInternalPass); + mgmtCreateUser(pAcct, "_root", tsInternalPass); + acctReleaseAcct(pAcct); + + return 0; +} + int32_t mgmtInitUsers() { SUserObj tObj; tsUserUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { + .tableId = SDB_TABLE_USER, .tableName = "users", .hashSessions = TSDB_MAX_USERS, .maxRowSize = tsUserUpdateSize, @@ -105,6 +111,7 @@ int32_t mgmtInitUsers() { .encodeFp = mgmtUserActionEncode, .decodeFp = mgmtUserActionDecode, .destroyFp = mgmtUserActionDestroy, + .updateAllFp = mgmtUserActionUpdateAll }; tsUserSdb = sdbOpenTable(&tableDesc); @@ -113,12 +120,6 @@ int32_t mgmtInitUsers() { return -1; } - SAcctObj *pAcct = acctGetAcct("root"); - mgmtCreateUser(pAcct, "root", "taosdata"); - mgmtCreateUser(pAcct, "monitor", tsInternalPass); - mgmtCreateUser(pAcct, "_root", tsInternalPass); - acctReleaseAcct(pAcct); - mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_USER, mgmtProcessCreateUserMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_ALTER_USER, mgmtProcessAlterUserMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_USER, mgmtProcessDropUserMsg); diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 98ae422904..09fdffb68e 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -138,13 +138,9 @@ static int32_t mgmtVgroupActionUpdate(SSdbOperDesc *pOper) { static int32_t mgmtVgroupActionEncode(SSdbOperDesc *pOper) { SVgObj *pVgroup = pOper->pObj; - if (pOper->maxRowSize < tsVgUpdateSize) { - return -1; - } else { - memcpy(pOper->rowData, pVgroup, tsVgUpdateSize); - pOper->rowSize = tsVgUpdateSize; - return TSDB_CODE_SUCCESS; - } + memcpy(pOper->rowData, pVgroup, tsVgUpdateSize); + pOper->rowSize = tsVgUpdateSize; + return TSDB_CODE_SUCCESS; } static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) { @@ -156,11 +152,16 @@ static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } +static int32_t mgmtVgroupActionUpdateAll() { + return 0; +} + int32_t mgmtInitVgroups() { SVgObj tObj; tsVgUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { + .tableId = SDB_TABLE_VGROUP, .tableName = "vgroups", .hashSessions = TSDB_MAX_VGROUPS, .maxRowSize = tsVgUpdateSize, @@ -172,6 +173,7 @@ int32_t mgmtInitVgroups() { .encodeFp = mgmtVgroupActionEncode, .decodeFp = mgmtVgroupActionDecode, .destroyFp = mgmtVgroupActionDestroy, + .updateAllFp = mgmtVgroupActionUpdateAll, }; tsVgroupSdb = sdbOpenTable(&tableDesc); @@ -187,6 +189,7 @@ int32_t mgmtInitVgroups() { mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mgmtProcessVnodeCfgMsg); mTrace("vgroup is initialized"); + return 0; } diff --git a/src/util/inc/cJSON.h b/src/util/inc/cJSON.h index 31c6d19e78..cdd5faa523 100644 --- a/src/util/inc/cJSON.h +++ b/src/util/inc/cJSON.h @@ -71,6 +71,9 @@ typedef struct cJSON /* The item's name string, if this item is the child of, or is in the list of subitems of an object. */ char *string; + + //Keep the original string of number + char numberstring[13]; } cJSON; typedef struct cJSON_Hooks diff --git a/src/util/src/cJSON.c b/src/util/src/cJSON.c index fa836f9871..ecf0e05b42 100644 --- a/src/util/src/cJSON.c +++ b/src/util/src/cJSON.c @@ -289,6 +289,9 @@ loop_end: item->type = cJSON_Number; input_buffer->offset += (size_t)(after_end - number_c_string); + + strncpy(item->numberstring, (const char *)number_c_string, 12); + return true; } diff --git a/tests/tsim/inc/cJSON.h b/tests/tsim/inc/cJSON.h deleted file mode 100644 index cdd5faa523..0000000000 --- a/tests/tsim/inc/cJSON.h +++ /dev/null @@ -1,267 +0,0 @@ -/* - Copyright (c) 2009-2017 Dave Gamble and cJSON contributors - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. -*/ - -#ifndef cJSON__h -#define cJSON__h - -#ifdef __cplusplus -extern "C" -{ -#endif - -/* project version */ -#define CJSON_VERSION_MAJOR 1 -#define CJSON_VERSION_MINOR 5 -#define CJSON_VERSION_PATCH 9 - -#include -#include - -/* cJSON Types: */ -#define cJSON_Invalid (0) -#define cJSON_False (1 << 0) -#define cJSON_True (1 << 1) -#define cJSON_NULL (1 << 2) -#define cJSON_Number (1 << 3) -#define cJSON_String (1 << 4) -#define cJSON_Array (1 << 5) -#define cJSON_Object (1 << 6) -#define cJSON_Raw (1 << 7) /* raw json */ - -#define cJSON_IsReference 256 -#define cJSON_StringIsConst 512 - -/* The cJSON structure: */ -typedef struct cJSON -{ - /* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */ - struct cJSON *next; - struct cJSON *prev; - /* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */ - struct cJSON *child; - - /* The type of the item, as above. */ - int type; - - /* The item's string, if type==cJSON_String and type == cJSON_Raw */ - char *valuestring; - /* writing to valueint is DEPRECATED, use cJSON_SetNumberValue instead */ - int64_t valueint; - /* The item's number, if type==cJSON_Number */ - double valuedouble; - - /* The item's name string, if this item is the child of, or is in the list of subitems of an object. */ - char *string; - - //Keep the original string of number - char numberstring[13]; -} cJSON; - -typedef struct cJSON_Hooks -{ - void *(*malloc_fn)(size_t sz); - void (*free_fn)(void *ptr); -} cJSON_Hooks; - -typedef int cJSON_bool; - -#if !defined(__WINDOWS__) && (defined(WIN32) || defined(WIN64) || defined(_MSC_VER) || defined(_WIN32)) -#define __WINDOWS__ -#endif -#ifdef __WINDOWS__ - -/* When compiling for windows, we specify a specific calling convention to avoid issues where we are being called from a project with a different default calling convention. For windows you have 2 define options: - -CJSON_HIDE_SYMBOLS - Define this in the case where you don't want to ever dllexport symbols -CJSON_EXPORT_SYMBOLS - Define this on library build when you want to dllexport symbols (default) -CJSON_IMPORT_SYMBOLS - Define this if you want to dllimport symbol - -For *nix builds that support visibility attribute, you can define similar behavior by - -setting default visibility to hidden by adding --fvisibility=hidden (for gcc) -or --xldscope=hidden (for sun cc) -to CFLAGS - -then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJSON_EXPORT_SYMBOLS does - -*/ - -/* export symbols by default, this is necessary for copy pasting the C and header file */ -#if !defined(CJSON_HIDE_SYMBOLS) && !defined(CJSON_IMPORT_SYMBOLS) && !defined(CJSON_EXPORT_SYMBOLS) -#define CJSON_EXPORT_SYMBOLS -#endif - -#if defined(CJSON_HIDE_SYMBOLS) -#define CJSON_PUBLIC(type) type __stdcall -#elif defined(CJSON_EXPORT_SYMBOLS) -#define CJSON_PUBLIC(type) __declspec(dllexport) type __stdcall -#elif defined(CJSON_IMPORT_SYMBOLS) -#define CJSON_PUBLIC(type) __declspec(dllimport) type __stdcall -#endif -#else /* !WIN32 */ -#if (defined(__GNUC__) || defined(__SUNPRO_CC) || defined (__SUNPRO_C)) && defined(CJSON_API_VISIBILITY) -#define CJSON_PUBLIC(type) __attribute__((visibility("default"))) type -#else -#define CJSON_PUBLIC(type) type -#endif -#endif - -/* Limits how deeply nested arrays/objects can be before cJSON rejects to parse them. - * This is to prevent stack overflows. */ -#ifndef CJSON_NESTING_LIMIT -#define CJSON_NESTING_LIMIT 1000 -#endif - -/* returns the version of cJSON as a string */ -CJSON_PUBLIC(const char*) cJSON_Version(void); - -/* Supply malloc, realloc and free functions to cJSON */ -CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks); - -/* Memory Management: the caller is always responsible to free the results from all variants of cJSON_Parse (with cJSON_Delete) and cJSON_Print (with stdlib free, cJSON_Hooks.free_fn, or cJSON_free as appropriate). The exception is cJSON_PrintPreallocated, where the caller has full responsibility of the buffer. */ -/* Supply a block of JSON, and this returns a cJSON object you can interrogate. */ -CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value); -/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */ -/* If you supply a ptr in return_parse_end and parsing fails, then return_parse_end will contain a pointer to the error so will match cJSON_GetErrorPtr(). */ -CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated); - -/* Render a cJSON entity to text for transfer/storage. */ -CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item); -/* Render a cJSON entity to text for transfer/storage without any formatting. */ -CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item); -/* Render a cJSON entity to text using a buffered strategy. prebuffer is a guess at the final size. guessing well reduces reallocation. fmt=0 gives unformatted, =1 gives formatted */ -CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt); -/* Render a cJSON entity to text using a buffer already allocated in memory with given length. Returns 1 on success and 0 on failure. */ -/* NOTE: cJSON is not always 100% accurate in estimating how much memory it will use, so to be safe allocate 5 bytes more than you actually need */ -CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buffer, const int length, const cJSON_bool format); -/* Delete a cJSON entity and all subentities. */ -CJSON_PUBLIC(void) cJSON_Delete(cJSON *c); - -/* Returns the number of items in an array (or object). */ -CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array); -/* Retrieve item number "item" from array "array". Returns NULL if unsuccessful. */ -CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index); -/* Get item "string" from object. Case insensitive. */ -CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string); -CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string); -CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string); -/* For analysing failed parses. This returns a pointer to the parse error. You'll probably need to look a few chars back to make sense of it. Defined when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */ -CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void); - -/* These functions check the type of an item */ -CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item); - -/* These calls create a cJSON item of the appropriate type. */ -CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void); -CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void); -CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void); -CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool boolean); -CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num); -CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string); -/* raw json */ -CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw); -CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void); -CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void); - -/* These utilities create an Array of count items. */ -CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count); -CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count); -CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count); -CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char **strings, int count); - -/* Append item to the specified array/object. */ -CJSON_PUBLIC(void) cJSON_AddItemToArray(cJSON *array, cJSON *item); -CJSON_PUBLIC(void) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item); -/* Use this when string is definitely const (i.e. a literal, or as good as), and will definitely survive the cJSON object. - * WARNING: When this function was used, make sure to always check that (item->type & cJSON_StringIsConst) is zero before - * writing to `item->string` */ -CJSON_PUBLIC(void) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item); -/* Append reference to item to the specified array/object. Use this when you want to add an existing cJSON to a new cJSON, but don't want to corrupt your existing cJSON. */ -CJSON_PUBLIC(void) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item); -CJSON_PUBLIC(void) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item); - -/* Remove/Detatch items from Arrays/Objects. */ -CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item); -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which); -CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which); -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string); -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string); -CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string); -CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string); - -/* Update array items. */ -CJSON_PUBLIC(void) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem); /* Shifts pre-existing items to the right. */ -CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement); -CJSON_PUBLIC(void) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem); -CJSON_PUBLIC(void) cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem); -CJSON_PUBLIC(void) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object,const char *string,cJSON *newitem); - -/* Duplicate a cJSON item */ -CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse); -/* Duplicate will create a new, identical cJSON item to the one you pass, in new memory that will -need to be released. With recurse!=0, it will duplicate any children connected to the item. -The item->next and ->prev pointers are always zero on return from Duplicate. */ -/* Recursively compare two cJSON items for equality. If either a or b is NULL or invalid, they will be considered unequal. - * case_sensitive determines if object keys are treated case sensitive (1) or case insensitive (0) */ -CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive); - - -CJSON_PUBLIC(void) cJSON_Minify(char *json); - -/* Macros for creating things quickly. */ -#define cJSON_AddNullToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateNull()) -#define cJSON_AddTrueToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateTrue()) -#define cJSON_AddFalseToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateFalse()) -#define cJSON_AddBoolToObject(object,name,b) cJSON_AddItemToObject(object, name, cJSON_CreateBool(b)) -#define cJSON_AddNumberToObject(object,name,n) cJSON_AddItemToObject(object, name, cJSON_CreateNumber(n)) -#define cJSON_AddStringToObject(object,name,s) cJSON_AddItemToObject(object, name, cJSON_CreateString(s)) -#define cJSON_AddRawToObject(object,name,s) cJSON_AddItemToObject(object, name, cJSON_CreateRaw(s)) - -/* When assigning an integer value, it needs to be propagated to valuedouble too. */ -#define cJSON_SetIntValue(object, number) ((object) ? (object)->valueint = (object)->valuedouble = (number) : (number)) -/* helper for the cJSON_SetNumberValue macro */ -CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number); -#define cJSON_SetNumberValue(object, number) ((object != NULL) ? cJSON_SetNumberHelper(object, (double)number) : (number)) - -/* Macro for iterating over an array or object */ -#define cJSON_ArrayForEach(element, array) for(element = (array != NULL) ? (array)->child : NULL; element != NULL; element = element->next) - -/* malloc/free objects using the malloc/free functions that have been set with cJSON_InitHooks */ -CJSON_PUBLIC(void *) cJSON_malloc(size_t size); -CJSON_PUBLIC(void) cJSON_free(void *object); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/tests/tsim/src/cJSON.c b/tests/tsim/src/cJSON.c deleted file mode 100644 index ecf0e05b42..0000000000 --- a/tests/tsim/src/cJSON.c +++ /dev/null @@ -1,2702 +0,0 @@ -/* - Copyright (c) 2009-2017 Dave Gamble and cJSON contributors - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. -*/ - -/* cJSON */ -/* JSON parser in C. */ - -#ifdef __GNUC__ -#pragma GCC visibility push(default) -#endif - -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __GNUC__ -#pragma GCC visibility pop -#endif - -#include "cJSON.h" - -/* define our own boolean type */ -#define true ((cJSON_bool)1) -#define false ((cJSON_bool)0) - -typedef struct { - const unsigned char *json; - size_t position; -} error; -static error global_error = { NULL, 0 }; - -CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void) -{ - return (const char*) (global_error.json + global_error.position); -} - -/* This is a safeguard to prevent copy-pasters from using incompatible C and header files */ -#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 5) || (CJSON_VERSION_PATCH != 9) - #error cJSON.h and cJSON.c have different versions. Make sure that both have the same. -#endif - -CJSON_PUBLIC(const char*) cJSON_Version(void) -{ - static char version[15]; - sprintf(version, "%i.%i.%i", CJSON_VERSION_MAJOR, CJSON_VERSION_MINOR, CJSON_VERSION_PATCH); - - return version; -} - -/* Case insensitive string comparison, doesn't consider two NULL pointers equal though */ -static int case_insensitive_strcmp(const unsigned char *string1, const unsigned char *string2) -{ - if ((string1 == NULL) || (string2 == NULL)) - { - return 1; - } - - if (string1 == string2) - { - return 0; - } - - for(; tolower(*string1) == tolower(*string2); (void)string1++, string2++) - { - if (*string1 == '\0') - { - return 0; - } - } - - return tolower(*string1) - tolower(*string2); -} - -typedef struct internal_hooks -{ - void *(*allocate)(size_t size); - void (*deallocate)(void *pointer); - void *(*reallocate)(void *pointer, size_t size); -} internal_hooks; - -static internal_hooks global_hooks = { malloc, free, realloc }; - -static unsigned char* cJSON_strdup(const unsigned char* string, const internal_hooks * const hooks) -{ - size_t length = 0; - unsigned char *copy = NULL; - - if (string == NULL) - { - return NULL; - } - - length = strlen((const char*)string) + sizeof(""); - if (!(copy = (unsigned char*)hooks->allocate(length))) - { - return NULL; - } - memcpy(copy, string, length); - - return copy; -} - -CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks) -{ - if (hooks == NULL) - { - /* Reset hooks */ - global_hooks.allocate = malloc; - global_hooks.deallocate = free; - global_hooks.reallocate = realloc; - return; - } - - global_hooks.allocate = malloc; - if (hooks->malloc_fn != NULL) - { - global_hooks.allocate = hooks->malloc_fn; - } - - global_hooks.deallocate = free; - if (hooks->free_fn != NULL) - { - global_hooks.deallocate = hooks->free_fn; - } - - /* use realloc only if both free and malloc are used */ - global_hooks.reallocate = NULL; - if ((global_hooks.allocate == malloc) && (global_hooks.deallocate == free)) - { - global_hooks.reallocate = realloc; - } -} - -/* Internal constructor. */ -static cJSON *cJSON_New_Item(const internal_hooks * const hooks) -{ - cJSON* node = (cJSON*)hooks->allocate(sizeof(cJSON)); - if (node) - { - memset(node, '\0', sizeof(cJSON)); - } - - return node; -} - -/* Delete a cJSON structure. */ -CJSON_PUBLIC(void) cJSON_Delete(cJSON *item) -{ - cJSON *next = NULL; - while (item != NULL) - { - next = item->next; - if (!(item->type & cJSON_IsReference) && (item->child != NULL)) - { - cJSON_Delete(item->child); - } - if (!(item->type & cJSON_IsReference) && (item->valuestring != NULL)) - { - global_hooks.deallocate(item->valuestring); - } - if (!(item->type & cJSON_StringIsConst) && (item->string != NULL)) - { - global_hooks.deallocate(item->string); - } - global_hooks.deallocate(item); - item = next; - } -} - -/* get the decimal point character of the current locale */ -static unsigned char get_decimal_point(void) -{ - struct lconv *lconv = localeconv(); - return (unsigned char) lconv->decimal_point[0]; -} - -typedef struct -{ - const unsigned char *content; - size_t length; - size_t offset; - size_t depth; /* How deeply nested (in arrays/objects) is the input at the current offset. */ - internal_hooks hooks; -} parse_buffer; - -/* check if the given size is left to read in a given parse buffer (starting with 1) */ -#define can_read(buffer, size) ((buffer != NULL) && (((buffer)->offset + size) <= (buffer)->length)) -#define cannot_read(buffer, size) (!can_read(buffer, size)) -/* check if the buffer can be accessed at the given index (starting with 0) */ -#define can_access_at_index(buffer, index) ((buffer != NULL) && (((buffer)->offset + index) < (buffer)->length)) -#define cannot_access_at_index(buffer, index) (!can_access_at_index(buffer, index)) -/* get a pointer to the buffer at the position */ -#define buffer_at_offset(buffer) ((buffer)->content + (buffer)->offset) - -/* Parse the input text to generate a number, and populate the result into item. */ -static cJSON_bool parse_number(cJSON * const item, parse_buffer * const input_buffer) -{ - double number = 0; - unsigned char *after_end = NULL; - unsigned char number_c_string[64]; - unsigned char decimal_point = get_decimal_point(); - size_t i = 0; - - if ((input_buffer == NULL) || (input_buffer->content == NULL)) - { - return false; - } - - /* copy the number into a temporary buffer and replace '.' with the decimal point - * of the current locale (for strtod) - * This also takes care of '\0' not necessarily being available for marking the end of the input */ - for (i = 0; (i < (sizeof(number_c_string) - 1)) && can_access_at_index(input_buffer, i); i++) - { - switch (buffer_at_offset(input_buffer)[i]) - { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - case '+': - case '-': - case 'e': - case 'E': - number_c_string[i] = buffer_at_offset(input_buffer)[i]; - break; - - case '.': - number_c_string[i] = decimal_point; - break; - - default: - goto loop_end; - } - } -loop_end: - number_c_string[i] = '\0'; - - number = strtod((const char*)number_c_string, (char**)&after_end); - if (number_c_string == after_end) - { - return false; /* parse_error */ - } - - item->valuedouble = number; - - /* use saturation in case of overflow */ - if (number >= LLONG_MAX) - { - item->valueint = LLONG_MAX; - } - else if (number <= LLONG_MIN) - { - item->valueint = LLONG_MIN; - } - else - { - item->valueint = (int64_t)number; - } - - item->type = cJSON_Number; - - input_buffer->offset += (size_t)(after_end - number_c_string); - - strncpy(item->numberstring, (const char *)number_c_string, 12); - - return true; -} - -/* don't ask me, but the original cJSON_SetNumberValue returns an integer or double */ -CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number) -{ - if (number >= LLONG_MAX) - { - object->valueint = LLONG_MAX; - } - else if (number <= LLONG_MIN) - { - object->valueint = LLONG_MIN; - } - else - { - object->valueint = (int64_t)number; - } - - return object->valuedouble = number; -} - -typedef struct -{ - unsigned char *buffer; - size_t length; - size_t offset; - size_t depth; /* current nesting depth (for formatted printing) */ - cJSON_bool noalloc; - cJSON_bool format; /* is this print a formatted print */ - internal_hooks hooks; -} printbuffer; - -/* realloc printbuffer if necessary to have at least "needed" bytes more */ -static unsigned char* ensure(printbuffer * const p, size_t needed) -{ - unsigned char *newbuffer = NULL; - size_t newsize = 0; - - if ((p == NULL) || (p->buffer == NULL)) - { - return NULL; - } - - if ((p->length > 0) && (p->offset >= p->length)) - { - /* make sure that offset is valid */ - return NULL; - } - - if (needed > LLONG_MAX) - { - /* sizes bigger than LLONG_MAX are currently not supported */ - return NULL; - } - - needed += p->offset + 1; - if (needed <= p->length) - { - return p->buffer + p->offset; - } - - if (p->noalloc) { - return NULL; - } - - /* calculate new buffer size */ - if (needed > (LLONG_MAX / 2)) - { - /* overflow of int, use LLONG_MAX if possible */ - if (needed <= LLONG_MAX) - { - newsize = LLONG_MAX; - } - else - { - return NULL; - } - } - else - { - newsize = needed * 2; - } - - if (p->hooks.reallocate != NULL) - { - /* reallocate with realloc if available */ - newbuffer = (unsigned char*)p->hooks.reallocate(p->buffer, newsize); - if (newbuffer == NULL) - { - p->hooks.deallocate(p->buffer); - p->length = 0; - p->buffer = NULL; - - return NULL; - } - } - else - { - /* otherwise reallocate manually */ - newbuffer = (unsigned char*)p->hooks.allocate(newsize); - if (!newbuffer) - { - p->hooks.deallocate(p->buffer); - p->length = 0; - p->buffer = NULL; - - return NULL; - } - if (newbuffer) - { - memcpy(newbuffer, p->buffer, p->offset + 1); - } - p->hooks.deallocate(p->buffer); - } - p->length = newsize; - p->buffer = newbuffer; - - return newbuffer + p->offset; -} - -/* calculate the new length of the string in a printbuffer and update the offset */ -static void update_offset(printbuffer * const buffer) -{ - const unsigned char *buffer_pointer = NULL; - if ((buffer == NULL) || (buffer->buffer == NULL)) - { - return; - } - buffer_pointer = buffer->buffer + buffer->offset; - - buffer->offset += strlen((const char*)buffer_pointer); -} - -/* Render the number nicely from the given item into a string. */ -static cJSON_bool print_number(const cJSON * const item, printbuffer * const output_buffer) -{ - unsigned char *output_pointer = NULL; - double d = item->valuedouble; - int length = 0; - size_t i = 0; - unsigned char number_buffer[26]; /* temporary buffer to print the number into */ - unsigned char decimal_point = get_decimal_point(); - double test; - - if (output_buffer == NULL) - { - return false; - } - - /* This checks for NaN and Infinity */ - if ((d * 0) != 0) - { - length = sprintf((char*)number_buffer, "null"); - } - else - { - /* Try 15 decimal places of precision to avoid nonsignificant nonzero digits */ - length = sprintf((char*)number_buffer, "%1.15g", d); - - /* Check whether the original double can be recovered */ - if ((sscanf((char*)number_buffer, "%lg", &test) != 1) || ((double)test != d)) - { - /* If not, print with 17 decimal places of precision */ - length = sprintf((char*)number_buffer, "%1.17g", d); - } - } - - /* sprintf failed or buffer overrun occured */ - if ((length < 0) || (length > (int)(sizeof(number_buffer) - 1))) - { - return false; - } - - /* reserve appropriate space in the output */ - output_pointer = ensure(output_buffer, (size_t)length); - if (output_pointer == NULL) - { - return false; - } - - /* copy the printed number to the output and replace locale - * dependent decimal point with '.' */ - for (i = 0; i < ((size_t)length); i++) - { - if (number_buffer[i] == decimal_point) - { - output_pointer[i] = '.'; - continue; - } - - output_pointer[i] = number_buffer[i]; - } - output_pointer[i] = '\0'; - - output_buffer->offset += (size_t)length; - - return true; -} - -/* parse 4 digit hexadecimal number */ -static unsigned parse_hex4(const unsigned char * const input) -{ - unsigned int h = 0; - size_t i = 0; - - for (i = 0; i < 4; i++) - { - /* parse digit */ - if ((input[i] >= '0') && (input[i] <= '9')) - { - h += (unsigned int) input[i] - '0'; - } - else if ((input[i] >= 'A') && (input[i] <= 'F')) - { - h += (unsigned int) 10 + input[i] - 'A'; - } - else if ((input[i] >= 'a') && (input[i] <= 'f')) - { - h += (unsigned int) 10 + input[i] - 'a'; - } - else /* invalid */ - { - return 0; - } - - if (i < 3) - { - /* shift left to make place for the next nibble */ - h = h << 4; - } - } - - return h; -} - -/* converts a UTF-16 literal to UTF-8 - * A literal can be one or two sequences of the form \uXXXX */ -static unsigned char utf16_literal_to_utf8(const unsigned char * const input_pointer, const unsigned char * const input_end, unsigned char **output_pointer) -{ - uint64_t codepoint = 0; - unsigned int first_code = 0; - const unsigned char *first_sequence = input_pointer; - unsigned char utf8_length = 0; - unsigned char utf8_position = 0; - unsigned char sequence_length = 0; - unsigned char first_byte_mark = 0; - - if ((input_end - first_sequence) < 6) - { - /* input ends unexpectedly */ - goto fail; - } - - /* get the first utf16 sequence */ - first_code = parse_hex4(first_sequence + 2); - - /* check that the code is valid */ - if (((first_code >= 0xDC00) && (first_code <= 0xDFFF))) - { - goto fail; - } - - /* UTF16 surrogate pair */ - if ((first_code >= 0xD800) && (first_code <= 0xDBFF)) - { - const unsigned char *second_sequence = first_sequence + 6; - unsigned int second_code = 0; - sequence_length = 12; /* \uXXXX\uXXXX */ - - if ((input_end - second_sequence) < 6) - { - /* input ends unexpectedly */ - goto fail; - } - - if ((second_sequence[0] != '\\') || (second_sequence[1] != 'u')) - { - /* missing second half of the surrogate pair */ - goto fail; - } - - /* get the second utf16 sequence */ - second_code = parse_hex4(second_sequence + 2); - /* check that the code is valid */ - if ((second_code < 0xDC00) || (second_code > 0xDFFF)) - { - /* invalid second half of the surrogate pair */ - goto fail; - } - - - /* calculate the unicode codepoint from the surrogate pair */ - codepoint = 0x10000 + (((first_code & 0x3FF) << 10) | (second_code & 0x3FF)); - } - else - { - sequence_length = 6; /* \uXXXX */ - codepoint = first_code; - } - - /* encode as UTF-8 - * takes at maximum 4 bytes to encode: - * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ - if (codepoint < 0x80) - { - /* normal ascii, encoding 0xxxxxxx */ - utf8_length = 1; - } - else if (codepoint < 0x800) - { - /* two bytes, encoding 110xxxxx 10xxxxxx */ - utf8_length = 2; - first_byte_mark = 0xC0; /* 11000000 */ - } - else if (codepoint < 0x10000) - { - /* three bytes, encoding 1110xxxx 10xxxxxx 10xxxxxx */ - utf8_length = 3; - first_byte_mark = 0xE0; /* 11100000 */ - } - else if (codepoint <= 0x10FFFF) - { - /* four bytes, encoding 1110xxxx 10xxxxxx 10xxxxxx 10xxxxxx */ - utf8_length = 4; - first_byte_mark = 0xF0; /* 11110000 */ - } - else - { - /* invalid unicode codepoint */ - goto fail; - } - - /* encode as utf8 */ - for (utf8_position = (unsigned char)(utf8_length - 1); utf8_position > 0; utf8_position--) - { - /* 10xxxxxx */ - (*output_pointer)[utf8_position] = (unsigned char)((codepoint | 0x80) & 0xBF); - codepoint >>= 6; - } - /* encode first byte */ - if (utf8_length > 1) - { - (*output_pointer)[0] = (unsigned char)((codepoint | first_byte_mark) & 0xFF); - } - else - { - (*output_pointer)[0] = (unsigned char)(codepoint & 0x7F); - } - - *output_pointer += utf8_length; - - return sequence_length; - -fail: - return 0; -} - -/* Parse the input text into an unescaped cinput, and populate item. */ -static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_buffer) -{ - const unsigned char *input_pointer = buffer_at_offset(input_buffer) + 1; - const unsigned char *input_end = buffer_at_offset(input_buffer) + 1; - unsigned char *output_pointer = NULL; - unsigned char *output = NULL; - - /* not a string */ - if (buffer_at_offset(input_buffer)[0] != '\"') - { - goto fail; - } - - { - /* calculate approximate size of the output (overestimate) */ - size_t allocation_length = 0; - size_t skipped_bytes = 0; - while (((size_t)(input_end - input_buffer->content) < input_buffer->length) && (*input_end != '\"')) - { - /* is escape sequence */ - if (input_end[0] == '\\') - { - if ((size_t)(input_end + 1 - input_buffer->content) >= input_buffer->length) - { - /* prevent buffer overflow when last input character is a backslash */ - goto fail; - } - skipped_bytes++; - input_end++; - } - input_end++; - } - if (((size_t)(input_end - input_buffer->content) >= input_buffer->length) || (*input_end != '\"')) - { - goto fail; /* string ended unexpectedly */ - } - - /* This is at most how much we need for the output */ - allocation_length = (size_t) (input_end - buffer_at_offset(input_buffer)) - skipped_bytes; - output = (unsigned char*)input_buffer->hooks.allocate(allocation_length + sizeof("")); - if (output == NULL) - { - goto fail; /* allocation failure */ - } - } - - output_pointer = output; - /* loop through the string literal */ - while (input_pointer < input_end) - { - if (*input_pointer != '\\') - { - *output_pointer++ = *input_pointer++; - } - /* escape sequence */ - else - { - unsigned char sequence_length = 2; - if ((input_end - input_pointer) < 1) - { - goto fail; - } - - switch (input_pointer[1]) - { - case 'b': - *output_pointer++ = '\b'; - break; - case 'f': - *output_pointer++ = '\f'; - break; - case 'n': - *output_pointer++ = '\n'; - break; - case 'r': - *output_pointer++ = '\r'; - break; - case 't': - *output_pointer++ = '\t'; - break; - case '\"': - case '\\': - case '/': - *output_pointer++ = input_pointer[1]; - break; - - /* UTF-16 literal */ - case 'u': - sequence_length = utf16_literal_to_utf8(input_pointer, input_end, &output_pointer); - if (sequence_length == 0) - { - /* failed to convert UTF16-literal to UTF-8 */ - goto fail; - } - break; - - default: - goto fail; - } - input_pointer += sequence_length; - } - } - - /* zero terminate the output */ - *output_pointer = '\0'; - - item->type = cJSON_String; - item->valuestring = (char*)output; - - input_buffer->offset = (size_t) (input_end - input_buffer->content); - input_buffer->offset++; - - return true; - -fail: - if (output != NULL) - { - input_buffer->hooks.deallocate(output); - } - - if (input_pointer != NULL) - { - input_buffer->offset = (size_t)(input_pointer - input_buffer->content); - } - - return false; -} - -/* Render the cstring provided to an escaped version that can be printed. */ -static cJSON_bool print_string_ptr(const unsigned char * const input, printbuffer * const output_buffer) -{ - const unsigned char *input_pointer = NULL; - unsigned char *output = NULL; - unsigned char *output_pointer = NULL; - size_t output_length = 0; - /* numbers of additional characters needed for escaping */ - size_t escape_characters = 0; - - if (output_buffer == NULL) - { - return false; - } - - /* empty string */ - if (input == NULL) - { - output = ensure(output_buffer, sizeof("\"\"")); - if (output == NULL) - { - return false; - } - strcpy((char*)output, "\"\""); - - return true; - } - - /* set "flag" to 1 if something needs to be escaped */ - for (input_pointer = input; *input_pointer; input_pointer++) - { - switch (*input_pointer) - { - case '\"': - case '\\': - case '\b': - case '\f': - case '\n': - case '\r': - case '\t': - /* one character escape sequence */ - escape_characters++; - break; - default: - if (*input_pointer < 32) - { - /* UTF-16 escape sequence uXXXX */ - escape_characters += 5; - } - break; - } - } - output_length = (size_t)(input_pointer - input) + escape_characters; - - output = ensure(output_buffer, output_length + sizeof("\"\"")); - if (output == NULL) - { - return false; - } - - /* no characters have to be escaped */ - if (escape_characters == 0) - { - output[0] = '\"'; - memcpy(output + 1, input, output_length); - output[output_length + 1] = '\"'; - output[output_length + 2] = '\0'; - - return true; - } - - output[0] = '\"'; - output_pointer = output + 1; - /* copy the string */ - for (input_pointer = input; *input_pointer != '\0'; (void)input_pointer++, output_pointer++) - { - if ((*input_pointer > 31) && (*input_pointer != '\"') && (*input_pointer != '\\')) - { - /* normal character, copy */ - *output_pointer = *input_pointer; - } - else - { - /* character needs to be escaped */ - *output_pointer++ = '\\'; - switch (*input_pointer) - { - case '\\': - *output_pointer = '\\'; - break; - case '\"': - *output_pointer = '\"'; - break; - case '\b': - *output_pointer = 'b'; - break; - case '\f': - *output_pointer = 'f'; - break; - case '\n': - *output_pointer = 'n'; - break; - case '\r': - *output_pointer = 'r'; - break; - case '\t': - *output_pointer = 't'; - break; - default: - /* escape and print as unicode codepoint */ - sprintf((char*)output_pointer, "u%04x", *input_pointer); - output_pointer += 4; - break; - } - } - } - output[output_length + 1] = '\"'; - output[output_length + 2] = '\0'; - - return true; -} - -/* Invoke print_string_ptr (which is useful) on an item. */ -static cJSON_bool print_string(const cJSON * const item, printbuffer * const p) -{ - return print_string_ptr((unsigned char*)item->valuestring, p); -} - -/* Predeclare these prototypes. */ -static cJSON_bool parse_value(cJSON * const item, parse_buffer * const input_buffer); -static cJSON_bool print_value(const cJSON * const item, printbuffer * const output_buffer); -static cJSON_bool parse_array(cJSON * const item, parse_buffer * const input_buffer); -static cJSON_bool print_array(const cJSON * const item, printbuffer * const output_buffer); -static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_buffer); -static cJSON_bool print_object(const cJSON * const item, printbuffer * const output_buffer); - -/* Utility to jump whitespace and cr/lf */ -static parse_buffer *buffer_skip_whitespace(parse_buffer * const buffer) -{ - if ((buffer == NULL) || (buffer->content == NULL)) - { - return NULL; - } - - while (can_access_at_index(buffer, 0) && (buffer_at_offset(buffer)[0] <= 32)) - { - buffer->offset++; - } - - if (buffer->offset == buffer->length) - { - buffer->offset--; - } - - return buffer; -} - -/* Parse an object - create a new root, and populate. */ -CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated) -{ - parse_buffer buffer = { 0, 0, 0, 0, { 0, 0, 0 } }; - cJSON *item = NULL; - - /* reset error position */ - global_error.json = NULL; - global_error.position = 0; - - if (value == NULL) - { - goto fail; - } - - buffer.content = (const unsigned char*)value; - buffer.length = strlen((const char*)value) + sizeof(""); - buffer.offset = 0; - buffer.hooks = global_hooks; - - item = cJSON_New_Item(&global_hooks); - if (item == NULL) /* memory fail */ - { - goto fail; - } - - if (!parse_value(item, buffer_skip_whitespace(&buffer))) - { - /* parse failure. ep is set. */ - goto fail; - } - - /* if we require null-terminated JSON without appended garbage, skip and then check for a null terminator */ - if (require_null_terminated) - { - buffer_skip_whitespace(&buffer); - if ((buffer.offset >= buffer.length) || buffer_at_offset(&buffer)[0] != '\0') - { - goto fail; - } - } - if (return_parse_end) - { - *return_parse_end = (const char*)buffer_at_offset(&buffer); - } - - return item; - -fail: - if (item != NULL) - { - cJSON_Delete(item); - } - - if (value != NULL) - { - error local_error; - local_error.json = (const unsigned char*)value; - local_error.position = 0; - - if (buffer.offset < buffer.length) - { - local_error.position = buffer.offset; - } - else if (buffer.length > 0) - { - local_error.position = buffer.length - 1; - } - - if (return_parse_end != NULL) - { - *return_parse_end = (const char*)local_error.json + local_error.position; - } - - global_error = local_error; - } - - return NULL; -} - -/* Default options for cJSON_Parse */ -CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value) -{ - return cJSON_ParseWithOpts(value, 0, 0); -} - -#define cjson_min(a, b) ((a < b) ? a : b) - -static unsigned char *print(const cJSON * const item, cJSON_bool format, const internal_hooks * const hooks) -{ - printbuffer buffer[1]; - unsigned char *printed = NULL; - - memset(buffer, 0, sizeof(buffer)); - - /* create buffer */ - buffer->buffer = (unsigned char*) hooks->allocate(256); - buffer->format = format; - buffer->hooks = *hooks; - if (buffer->buffer == NULL) - { - goto fail; - } - - /* print the value */ - if (!print_value(item, buffer)) - { - goto fail; - } - update_offset(buffer); - - /* check if reallocate is available */ - if (hooks->reallocate != NULL) - { - printed = (unsigned char*) hooks->reallocate(buffer->buffer, buffer->length); - buffer->buffer = NULL; - if (printed == NULL) { - goto fail; - } - } - else /* otherwise copy the JSON over to a new buffer */ - { - printed = (unsigned char*) hooks->allocate(buffer->offset + 1); - if (printed == NULL) - { - goto fail; - } - memcpy(printed, buffer->buffer, cjson_min(buffer->length, buffer->offset + 1)); - printed[buffer->offset] = '\0'; /* just to be sure */ - - /* free the buffer */ - hooks->deallocate(buffer->buffer); - } - - return printed; - -fail: - if (buffer->buffer != NULL) - { - hooks->deallocate(buffer->buffer); - } - - if (printed != NULL) - { - hooks->deallocate(printed); - } - - return NULL; -} - -/* Render a cJSON item/entity/structure to text. */ -CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item) -{ - return (char*)print(item, true, &global_hooks); -} - -CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item) -{ - return (char*)print(item, false, &global_hooks); -} - -CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt) -{ - printbuffer p = { 0, 0, 0, 0, 0, 0, { 0, 0, 0 } }; - - if (prebuffer < 0) - { - return NULL; - } - - p.buffer = (unsigned char*)global_hooks.allocate((size_t)prebuffer); - if (!p.buffer) - { - return NULL; - } - - p.length = (size_t)prebuffer; - p.offset = 0; - p.noalloc = false; - p.format = fmt; - p.hooks = global_hooks; - - if (!print_value(item, &p)) - { - global_hooks.deallocate(p.buffer); - return NULL; - } - - return (char*)p.buffer; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buf, const int len, const cJSON_bool fmt) -{ - printbuffer p = { 0, 0, 0, 0, 0, 0, { 0, 0, 0 } }; - - if ((len < 0) || (buf == NULL)) - { - return false; - } - - p.buffer = (unsigned char*)buf; - p.length = (size_t)len; - p.offset = 0; - p.noalloc = true; - p.format = fmt; - p.hooks = global_hooks; - - return print_value(item, &p); -} - -/* Parser core - when encountering text, process appropriately. */ -static cJSON_bool parse_value(cJSON * const item, parse_buffer * const input_buffer) -{ - if ((input_buffer == NULL) || (input_buffer->content == NULL)) - { - return false; /* no input */ - } - - /* parse the different types of values */ - /* null */ - if (can_read(input_buffer, 4) && (strncmp((const char*)buffer_at_offset(input_buffer), "null", 4) == 0)) - { - item->type = cJSON_NULL; - input_buffer->offset += 4; - return true; - } - /* false */ - if (can_read(input_buffer, 5) && (strncmp((const char*)buffer_at_offset(input_buffer), "false", 5) == 0)) - { - item->type = cJSON_False; - input_buffer->offset += 5; - return true; - } - /* true */ - if (can_read(input_buffer, 4) && (strncmp((const char*)buffer_at_offset(input_buffer), "true", 4) == 0)) - { - item->type = cJSON_True; - item->valueint = 1; - input_buffer->offset += 4; - return true; - } - /* string */ - if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '\"')) - { - return parse_string(item, input_buffer); - } - /* number */ - if (can_access_at_index(input_buffer, 0) && ((buffer_at_offset(input_buffer)[0] == '-') || ((buffer_at_offset(input_buffer)[0] >= '0') && (buffer_at_offset(input_buffer)[0] <= '9')))) - { - return parse_number(item, input_buffer); - } - /* array */ - if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '[')) - { - return parse_array(item, input_buffer); - } - /* object */ - if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '{')) - { - return parse_object(item, input_buffer); - } - - - return false; -} - -/* Render a value to text. */ -static cJSON_bool print_value(const cJSON * const item, printbuffer * const output_buffer) -{ - unsigned char *output = NULL; - - if ((item == NULL) || (output_buffer == NULL)) - { - return false; - } - - switch ((item->type) & 0xFF) - { - case cJSON_NULL: - output = ensure(output_buffer, 5); - if (output == NULL) - { - return false; - } - strcpy((char*)output, "null"); - return true; - - case cJSON_False: - output = ensure(output_buffer, 6); - if (output == NULL) - { - return false; - } - strcpy((char*)output, "false"); - return true; - - case cJSON_True: - output = ensure(output_buffer, 5); - if (output == NULL) - { - return false; - } - strcpy((char*)output, "true"); - return true; - - case cJSON_Number: - return print_number(item, output_buffer); - - case cJSON_Raw: - { - size_t raw_length = 0; - if (item->valuestring == NULL) - { - if (!output_buffer->noalloc) - { - output_buffer->hooks.deallocate(output_buffer->buffer); - } - return false; - } - - raw_length = strlen(item->valuestring) + sizeof(""); - output = ensure(output_buffer, raw_length); - if (output == NULL) - { - return false; - } - memcpy(output, item->valuestring, raw_length); - return true; - } - - case cJSON_String: - return print_string(item, output_buffer); - - case cJSON_Array: - return print_array(item, output_buffer); - - case cJSON_Object: - return print_object(item, output_buffer); - - default: - return false; - } -} - -/* Build an array from input text. */ -static cJSON_bool parse_array(cJSON * const item, parse_buffer * const input_buffer) -{ - cJSON *head = NULL; /* head of the linked list */ - cJSON *current_item = NULL; - - if (input_buffer->depth >= CJSON_NESTING_LIMIT) - { - return false; /* to deeply nested */ - } - input_buffer->depth++; - - if (buffer_at_offset(input_buffer)[0] != '[') - { - /* not an array */ - goto fail; - } - - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ']')) - { - /* empty array */ - goto success; - } - - /* check if we skipped to the end of the buffer */ - if (cannot_access_at_index(input_buffer, 0)) - { - input_buffer->offset--; - goto fail; - } - - /* step back to character in front of the first element */ - input_buffer->offset--; - /* loop through the comma separated array elements */ - do - { - /* allocate next item */ - cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks)); - if (new_item == NULL) - { - goto fail; /* allocation failure */ - } - - /* attach next item to list */ - if (head == NULL) - { - /* start the linked list */ - current_item = head = new_item; - } - else - { - /* add to the end and advance */ - current_item->next = new_item; - new_item->prev = current_item; - current_item = new_item; - } - - /* parse next value */ - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (!parse_value(current_item, input_buffer)) - { - goto fail; /* failed to parse value */ - } - buffer_skip_whitespace(input_buffer); - } - while (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ',')); - - if (cannot_access_at_index(input_buffer, 0) || buffer_at_offset(input_buffer)[0] != ']') - { - goto fail; /* expected end of array */ - } - -success: - input_buffer->depth--; - - item->type = cJSON_Array; - item->child = head; - - input_buffer->offset++; - - return true; - -fail: - if (head != NULL) - { - cJSON_Delete(head); - } - - return false; -} - -/* Render an array to text */ -static cJSON_bool print_array(const cJSON * const item, printbuffer * const output_buffer) -{ - unsigned char *output_pointer = NULL; - size_t length = 0; - cJSON *current_element = item->child; - - if (output_buffer == NULL) - { - return false; - } - - /* Compose the output array. */ - /* opening square bracket */ - output_pointer = ensure(output_buffer, 1); - if (output_pointer == NULL) - { - return false; - } - - *output_pointer = '['; - output_buffer->offset++; - output_buffer->depth++; - - while (current_element != NULL) - { - if (!print_value(current_element, output_buffer)) - { - return false; - } - update_offset(output_buffer); - if (current_element->next) - { - length = (size_t) (output_buffer->format ? 2 : 1); - output_pointer = ensure(output_buffer, length + 1); - if (output_pointer == NULL) - { - return false; - } - *output_pointer++ = ','; - if(output_buffer->format) - { - *output_pointer++ = ' '; - } - *output_pointer = '\0'; - output_buffer->offset += length; - } - current_element = current_element->next; - } - - output_pointer = ensure(output_buffer, 2); - if (output_pointer == NULL) - { - return false; - } - *output_pointer++ = ']'; - *output_pointer = '\0'; - output_buffer->depth--; - - return true; -} - -/* Build an object from the text. */ -static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_buffer) -{ - cJSON *head = NULL; /* linked list head */ - cJSON *current_item = NULL; - - if (input_buffer->depth >= CJSON_NESTING_LIMIT) - { - return false; /* to deeply nested */ - } - input_buffer->depth++; - - if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != '{')) - { - goto fail; /* not an object */ - } - - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '}')) - { - goto success; /* empty object */ - } - - /* check if we skipped to the end of the buffer */ - if (cannot_access_at_index(input_buffer, 0)) - { - input_buffer->offset--; - goto fail; - } - - /* step back to character in front of the first element */ - input_buffer->offset--; - /* loop through the comma separated array elements */ - do - { - /* allocate next item */ - cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks)); - if (new_item == NULL) - { - goto fail; /* allocation failure */ - } - - /* attach next item to list */ - if (head == NULL) - { - /* start the linked list */ - current_item = head = new_item; - } - else - { - /* add to the end and advance */ - current_item->next = new_item; - new_item->prev = current_item; - current_item = new_item; - } - - /* parse the name of the child */ - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (!parse_string(current_item, input_buffer)) - { - goto fail; /* faile to parse name */ - } - buffer_skip_whitespace(input_buffer); - - /* swap valuestring and string, because we parsed the name */ - current_item->string = current_item->valuestring; - current_item->valuestring = NULL; - - if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != ':')) - { - goto fail; /* invalid object */ - } - - /* parse the value */ - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (!parse_value(current_item, input_buffer)) - { - goto fail; /* failed to parse value */ - } - buffer_skip_whitespace(input_buffer); - } - while (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ',')); - - if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != '}')) - { - goto fail; /* expected end of object */ - } - -success: - input_buffer->depth--; - - item->type = cJSON_Object; - item->child = head; - - input_buffer->offset++; - return true; - -fail: - if (head != NULL) - { - cJSON_Delete(head); - } - - return false; -} - -/* Render an object to text. */ -static cJSON_bool print_object(const cJSON * const item, printbuffer * const output_buffer) -{ - unsigned char *output_pointer = NULL; - size_t length = 0; - cJSON *current_item = item->child; - - if (output_buffer == NULL) - { - return false; - } - - /* Compose the output: */ - length = (size_t) (output_buffer->format ? 2 : 1); /* fmt: {\n */ - output_pointer = ensure(output_buffer, length + 1); - if (output_pointer == NULL) - { - return false; - } - - *output_pointer++ = '{'; - output_buffer->depth++; - if (output_buffer->format) - { - *output_pointer++ = '\n'; - } - output_buffer->offset += length; - - while (current_item) - { - if (output_buffer->format) - { - size_t i; - output_pointer = ensure(output_buffer, output_buffer->depth); - if (output_pointer == NULL) - { - return false; - } - for (i = 0; i < output_buffer->depth; i++) - { - *output_pointer++ = '\t'; - } - output_buffer->offset += output_buffer->depth; - } - - /* print key */ - if (!print_string_ptr((unsigned char*)current_item->string, output_buffer)) - { - return false; - } - update_offset(output_buffer); - - length = (size_t) (output_buffer->format ? 2 : 1); - output_pointer = ensure(output_buffer, length); - if (output_pointer == NULL) - { - return false; - } - *output_pointer++ = ':'; - if (output_buffer->format) - { - *output_pointer++ = '\t'; - } - output_buffer->offset += length; - - /* print value */ - if (!print_value(current_item, output_buffer)) - { - return false; - } - update_offset(output_buffer); - - /* print comma if not last */ - length = (size_t) ((output_buffer->format ? 1 : 0) + (current_item->next ? 1 : 0)); - output_pointer = ensure(output_buffer, length + 1); - if (output_pointer == NULL) - { - return false; - } - if (current_item->next) - { - *output_pointer++ = ','; - } - - if (output_buffer->format) - { - *output_pointer++ = '\n'; - } - *output_pointer = '\0'; - output_buffer->offset += length; - - current_item = current_item->next; - } - - output_pointer = ensure(output_buffer, output_buffer->format ? (output_buffer->depth + 1) : 2); - if (output_pointer == NULL) - { - return false; - } - if (output_buffer->format) - { - size_t i; - for (i = 0; i < (output_buffer->depth - 1); i++) - { - *output_pointer++ = '\t'; - } - } - *output_pointer++ = '}'; - *output_pointer = '\0'; - output_buffer->depth--; - - return true; -} - -/* Get Array size/item / object item. */ -CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array) -{ - cJSON *child = NULL; - size_t size = 0; - - if (array == NULL) - { - return 0; - } - - child = array->child; - - while(child != NULL) - { - size++; - child = child->next; - } - - /* FIXME: Can overflow here. Cannot be fixed without breaking the API */ - - return (int)size; -} - -static cJSON* get_array_item(const cJSON *array, size_t index) -{ - cJSON *current_child = NULL; - - if (array == NULL) - { - return NULL; - } - - current_child = array->child; - while ((current_child != NULL) && (index > 0)) - { - index--; - current_child = current_child->next; - } - - return current_child; -} - -CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index) -{ - if (index < 0) - { - return NULL; - } - - return get_array_item(array, (size_t)index); -} - -static cJSON *get_object_item(const cJSON * const object, const char * const name, const cJSON_bool case_sensitive) -{ - cJSON *current_element = NULL; - - if ((object == NULL) || (name == NULL)) - { - return NULL; - } - - current_element = object->child; - if (case_sensitive) - { - while ((current_element != NULL) && (strcmp(name, current_element->string) != 0)) - { - current_element = current_element->next; - } - } - else - { - while ((current_element != NULL) && (case_insensitive_strcmp((const unsigned char*)name, (const unsigned char*)(current_element->string)) != 0)) - { - current_element = current_element->next; - } - } - - return current_element; -} - -CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string) -{ - return get_object_item(object, string, false); -} - -CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string) -{ - return get_object_item(object, string, true); -} - -CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string) -{ - return cJSON_GetObjectItem(object, string) ? 1 : 0; -} - -/* Utility for array list handling. */ -static void suffix_object(cJSON *prev, cJSON *item) -{ - prev->next = item; - item->prev = prev; -} - -/* Utility for handling references. */ -static cJSON *create_reference(const cJSON *item, const internal_hooks * const hooks) -{ - cJSON *reference = NULL; - if (item == NULL) - { - return NULL; - } - - reference = cJSON_New_Item(hooks); - if (reference == NULL) - { - return NULL; - } - - memcpy(reference, item, sizeof(cJSON)); - reference->string = NULL; - reference->type |= cJSON_IsReference; - reference->next = reference->prev = NULL; - return reference; -} - -/* Add item to array/object. */ -CJSON_PUBLIC(void) cJSON_AddItemToArray(cJSON *array, cJSON *item) -{ - cJSON *child = NULL; - - if ((item == NULL) || (array == NULL)) - { - return; - } - - child = array->child; - - if (child == NULL) - { - /* list is empty, start new one */ - array->child = item; - } - else - { - /* append to the end */ - while (child->next) - { - child = child->next; - } - suffix_object(child, item); - } -} - -CJSON_PUBLIC(void) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item) -{ - if (item == NULL) - { - return; - } - - /* call cJSON_AddItemToObjectCS for code reuse */ - cJSON_AddItemToObjectCS(object, (char*)cJSON_strdup((const unsigned char*)string, &global_hooks), item); - /* remove cJSON_StringIsConst flag */ - item->type &= ~cJSON_StringIsConst; -} - -#if defined (__clang__) || ((__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))) - #pragma GCC diagnostic push -#endif -#ifdef __GNUC__ -#pragma GCC diagnostic ignored "-Wcast-qual" -#endif - -/* Add an item to an object with constant string as key */ -CJSON_PUBLIC(void) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item) -{ - if ((item == NULL) || (string == NULL)) - { - return; - } - if (!(item->type & cJSON_StringIsConst) && item->string) - { - global_hooks.deallocate(item->string); - } - item->string = (char*)string; - item->type |= cJSON_StringIsConst; - cJSON_AddItemToArray(object, item); -} -#if defined (__clang__) || ((__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))) - #pragma GCC diagnostic pop -#endif - -CJSON_PUBLIC(void) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item) -{ - if (array == NULL) - { - return; - } - - cJSON_AddItemToArray(array, create_reference(item, &global_hooks)); -} - -CJSON_PUBLIC(void) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item) -{ - if ((object == NULL) || (string == NULL)) - { - return; - } - - cJSON_AddItemToObject(object, string, create_reference(item, &global_hooks)); -} - -CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item) -{ - if ((parent == NULL) || (item == NULL)) - { - return NULL; - } - - if (item->prev != NULL) - { - /* not the first element */ - item->prev->next = item->next; - } - if (item->next != NULL) - { - /* not the last element */ - item->next->prev = item->prev; - } - - if (item == parent->child) - { - /* first element */ - parent->child = item->next; - } - /* make sure the detached item doesn't point anywhere anymore */ - item->prev = NULL; - item->next = NULL; - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which) -{ - if (which < 0) - { - return NULL; - } - - return cJSON_DetachItemViaPointer(array, get_array_item(array, (size_t)which)); -} - -CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which) -{ - cJSON_Delete(cJSON_DetachItemFromArray(array, which)); -} - -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string) -{ - cJSON *to_detach = cJSON_GetObjectItem(object, string); - - return cJSON_DetachItemViaPointer(object, to_detach); -} - -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string) -{ - cJSON *to_detach = cJSON_GetObjectItemCaseSensitive(object, string); - - return cJSON_DetachItemViaPointer(object, to_detach); -} - -CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string) -{ - cJSON_Delete(cJSON_DetachItemFromObject(object, string)); -} - -CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string) -{ - cJSON_Delete(cJSON_DetachItemFromObjectCaseSensitive(object, string)); -} - -/* Replace array/object items with new ones. */ -CJSON_PUBLIC(void) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem) -{ - cJSON *after_inserted = NULL; - - if (which < 0) - { - return; - } - - after_inserted = get_array_item(array, (size_t)which); - if (after_inserted == NULL) - { - cJSON_AddItemToArray(array, newitem); - return; - } - - newitem->next = after_inserted; - newitem->prev = after_inserted->prev; - after_inserted->prev = newitem; - if (after_inserted == array->child) - { - array->child = newitem; - } - else - { - newitem->prev->next = newitem; - } -} - -CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement) -{ - if ((parent == NULL) || (replacement == NULL) || (item == NULL)) - { - return false; - } - - if (replacement == item) - { - return true; - } - - replacement->next = item->next; - replacement->prev = item->prev; - - if (replacement->next != NULL) - { - replacement->next->prev = replacement; - } - if (replacement->prev != NULL) - { - replacement->prev->next = replacement; - } - if (parent->child == item) - { - parent->child = replacement; - } - - item->next = NULL; - item->prev = NULL; - cJSON_Delete(item); - - return true; -} - -CJSON_PUBLIC(void) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem) -{ - if (which < 0) - { - return; - } - - cJSON_ReplaceItemViaPointer(array, get_array_item(array, (size_t)which), newitem); -} - -static cJSON_bool replace_item_in_object(cJSON *object, const char *string, cJSON *replacement, cJSON_bool case_sensitive) -{ - if ((replacement == NULL) || (string == NULL)) - { - return false; - } - - /* replace the name in the replacement */ - if (!(replacement->type & cJSON_StringIsConst) && (replacement->string != NULL)) - { - cJSON_free(replacement->string); - } - replacement->string = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks); - replacement->type &= ~cJSON_StringIsConst; - - cJSON_ReplaceItemViaPointer(object, get_object_item(object, string, case_sensitive), replacement); - - return true; -} - -CJSON_PUBLIC(void) cJSON_ReplaceItemInObject(cJSON *object, const char *string, cJSON *newitem) -{ - replace_item_in_object(object, string, newitem, false); -} - -CJSON_PUBLIC(void) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object, const char *string, cJSON *newitem) -{ - replace_item_in_object(object, string, newitem, true); -} - -/* Create basic types: */ -CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_NULL; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_True; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_False; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool b) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = b ? cJSON_True : cJSON_False; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_Number; - item->valuedouble = num; - - /* use saturation in case of overflow */ - if (num >= LLONG_MAX) - { - item->valueint = LLONG_MAX; - } - else if (num <= LLONG_MIN) - { - item->valueint = LLONG_MIN; - } - else - { - item->valueint = (int64_t)num; - } - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_String; - item->valuestring = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks); - if(!item->valuestring) - { - cJSON_Delete(item); - return NULL; - } - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_Raw; - item->valuestring = (char*)cJSON_strdup((const unsigned char*)raw, &global_hooks); - if(!item->valuestring) - { - cJSON_Delete(item); - return NULL; - } - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type=cJSON_Array; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if (item) - { - item->type = cJSON_Object; - } - - return item; -} - -/* Create Arrays: */ -CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count) -{ - size_t i = 0; - cJSON *n = NULL; - cJSON *p = NULL; - cJSON *a = NULL; - - if ((count < 0) || (numbers == NULL)) - { - return NULL; - } - - a = cJSON_CreateArray(); - for(i = 0; a && (i < (size_t)count); i++) - { - n = cJSON_CreateNumber(numbers[i]); - if (!n) - { - cJSON_Delete(a); - return NULL; - } - if(!i) - { - a->child = n; - } - else - { - suffix_object(p, n); - } - p = n; - } - - return a; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count) -{ - size_t i = 0; - cJSON *n = NULL; - cJSON *p = NULL; - cJSON *a = NULL; - - if ((count < 0) || (numbers == NULL)) - { - return NULL; - } - - a = cJSON_CreateArray(); - - for(i = 0; a && (i < (size_t)count); i++) - { - n = cJSON_CreateNumber((double)numbers[i]); - if(!n) - { - cJSON_Delete(a); - return NULL; - } - if(!i) - { - a->child = n; - } - else - { - suffix_object(p, n); - } - p = n; - } - - return a; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count) -{ - size_t i = 0; - cJSON *n = NULL; - cJSON *p = NULL; - cJSON *a = NULL; - - if ((count < 0) || (numbers == NULL)) - { - return NULL; - } - - a = cJSON_CreateArray(); - - for(i = 0;a && (i < (size_t)count); i++) - { - n = cJSON_CreateNumber(numbers[i]); - if(!n) - { - cJSON_Delete(a); - return NULL; - } - if(!i) - { - a->child = n; - } - else - { - suffix_object(p, n); - } - p = n; - } - - return a; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char **strings, int count) -{ - size_t i = 0; - cJSON *n = NULL; - cJSON *p = NULL; - cJSON *a = NULL; - - if ((count < 0) || (strings == NULL)) - { - return NULL; - } - - a = cJSON_CreateArray(); - - for (i = 0; a && (i < (size_t)count); i++) - { - n = cJSON_CreateString(strings[i]); - if(!n) - { - cJSON_Delete(a); - return NULL; - } - if(!i) - { - a->child = n; - } - else - { - suffix_object(p,n); - } - p = n; - } - - return a; -} - -/* Duplication */ -CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse) -{ - cJSON *newitem = NULL; - cJSON *child = NULL; - cJSON *next = NULL; - cJSON *newchild = NULL; - - /* Bail on bad ptr */ - if (!item) - { - goto fail; - } - /* Create new item */ - newitem = cJSON_New_Item(&global_hooks); - if (!newitem) - { - goto fail; - } - /* Copy over all vars */ - newitem->type = item->type & (~cJSON_IsReference); - newitem->valueint = item->valueint; - newitem->valuedouble = item->valuedouble; - if (item->valuestring) - { - newitem->valuestring = (char*)cJSON_strdup((unsigned char*)item->valuestring, &global_hooks); - if (!newitem->valuestring) - { - goto fail; - } - } - if (item->string) - { - newitem->string = (item->type&cJSON_StringIsConst) ? item->string : (char*)cJSON_strdup((unsigned char*)item->string, &global_hooks); - if (!newitem->string) - { - goto fail; - } - } - /* If non-recursive, then we're done! */ - if (!recurse) - { - return newitem; - } - /* Walk the ->next chain for the child. */ - child = item->child; - while (child != NULL) - { - newchild = cJSON_Duplicate(child, true); /* Duplicate (with recurse) each item in the ->next chain */ - if (!newchild) - { - goto fail; - } - if (next != NULL) - { - /* If newitem->child already set, then crosswire ->prev and ->next and move on */ - next->next = newchild; - newchild->prev = next; - next = newchild; - } - else - { - /* Set newitem->child and move to it */ - newitem->child = newchild; - next = newchild; - } - child = child->next; - } - - return newitem; - -fail: - if (newitem != NULL) - { - cJSON_Delete(newitem); - } - - return NULL; -} - -CJSON_PUBLIC(void) cJSON_Minify(char *json) -{ - unsigned char *into = (unsigned char*)json; - - if (json == NULL) - { - return; - } - - while (*json) - { - if (*json == ' ') - { - json++; - } - else if (*json == '\t') - { - /* Whitespace characters. */ - json++; - } - else if (*json == '\r') - { - json++; - } - else if (*json=='\n') - { - json++; - } - else if ((*json == '/') && (json[1] == '/')) - { - /* double-slash comments, to end of line. */ - while (*json && (*json != '\n')) - { - json++; - } - } - else if ((*json == '/') && (json[1] == '*')) - { - /* multiline comments. */ - while (*json && !((*json == '*') && (json[1] == '/'))) - { - json++; - } - json += 2; - } - else if (*json == '\"') - { - /* string literals, which are \" sensitive. */ - *into++ = (unsigned char)*json++; - while (*json && (*json != '\"')) - { - if (*json == '\\') - { - *into++ = (unsigned char)*json++; - } - *into++ = (unsigned char)*json++; - } - *into++ = (unsigned char)*json++; - } - else - { - /* All other characters. */ - *into++ = (unsigned char)*json++; - } - } - - /* and null-terminate. */ - *into = '\0'; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_Invalid; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_False; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xff) == cJSON_True; -} - - -CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & (cJSON_True | cJSON_False)) != 0; -} -CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_NULL; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_Number; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_String; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_Array; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_Object; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_Raw; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive) -{ - if ((a == NULL) || (b == NULL) || ((a->type & 0xFF) != (b->type & 0xFF)) || cJSON_IsInvalid(a)) - { - return false; - } - - /* check if type is valid */ - switch (a->type & 0xFF) - { - case cJSON_False: - case cJSON_True: - case cJSON_NULL: - case cJSON_Number: - case cJSON_String: - case cJSON_Raw: - case cJSON_Array: - case cJSON_Object: - break; - - default: - return false; - } - - /* identical objects are equal */ - if (a == b) - { - return true; - } - - switch (a->type & 0xFF) - { - /* in these cases and equal type is enough */ - case cJSON_False: - case cJSON_True: - case cJSON_NULL: - return true; - - case cJSON_Number: - if (a->valuedouble == b->valuedouble) - { - return true; - } - return false; - - case cJSON_String: - case cJSON_Raw: - if ((a->valuestring == NULL) || (b->valuestring == NULL)) - { - return false; - } - if (strcmp(a->valuestring, b->valuestring) == 0) - { - return true; - } - - return false; - - case cJSON_Array: - { - cJSON *a_element = a->child; - cJSON *b_element = b->child; - - for (; (a_element != NULL) && (b_element != NULL);) - { - if (!cJSON_Compare(a_element, b_element, case_sensitive)) - { - return false; - } - - a_element = a_element->next; - b_element = b_element->next; - } - - /* one of the arrays is longer than the other */ - if (a_element != b_element) { - return false; - } - - return true; - } - - case cJSON_Object: - { - cJSON *a_element = NULL; - cJSON *b_element = NULL; - cJSON_ArrayForEach(a_element, a) - { - /* TODO This has O(n^2) runtime, which is horrible! */ - b_element = get_object_item(b, a_element->string, case_sensitive); - if (b_element == NULL) - { - return false; - } - - if (!cJSON_Compare(a_element, b_element, case_sensitive)) - { - return false; - } - } - - /* doing this twice, once on a and b to prevent true comparison if a subset of b - * TODO: Do this the proper way, this is just a fix for now */ - cJSON_ArrayForEach(b_element, b) - { - a_element = get_object_item(a, b_element->string, case_sensitive); - if (a_element == NULL) - { - return false; - } - - if (!cJSON_Compare(b_element, a_element, case_sensitive)) - { - return false; - } - } - - return true; - } - - default: - return false; - } -} - -CJSON_PUBLIC(void *) cJSON_malloc(size_t size) -{ - return global_hooks.allocate(size); -} - -CJSON_PUBLIC(void) cJSON_free(void *object) -{ - global_hooks.deallocate(object); -} From ea6898a26de3c92e7c98d5d122af9434d345b4db Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 13 Apr 2020 22:59:40 +0800 Subject: [PATCH 75/78] [td-98] fix time range query bug when data kept in buffer --- src/query/src/queryExecutor.c | 27 +++++---- src/util/src/tcompare.c | 7 ++- src/util/src/tskiplist.c | 52 ++--------------- src/vnode/tsdb/src/tsdbRead.c | 103 ++++++++++++++++++++++++---------- 4 files changed, 98 insertions(+), 91 deletions(-) diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 8015360919..9607157bda 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -362,10 +362,10 @@ bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].pBase.functio bool doRevisedResultsByLimit(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - if ((pQuery->limit.limit > 0) && (pQuery->rec.rows + pQuery->rec.rows > pQuery->limit.limit)) { - pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.rows; - - // query completed + if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) { + pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.total; + assert(pQuery->rec.rows > 0); + setQueryStatus(pQuery, QUERY_COMPLETED); return true; } @@ -2503,17 +2503,20 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl } int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { - int firstPos, lastPos, midPos = -1; - int numOfPoints; - TSKEY *keyList; + int32_t midPos = -1; + int32_t numOfPoints; - if (num <= 0) return -1; + if (num <= 0) { + return -1; + } - keyList = (TSKEY *)pValue; - firstPos = 0; - lastPos = num - 1; + assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + + TSKEY* keyList = (TSKEY *)pValue; + int32_t firstPos = 0; + int32_t lastPos = num - 1; - if (order == 0) { + if (order == TSDB_ORDER_DESC) { // find the first position which is smaller than the key while (1) { if (key >= keyList[lastPos]) return lastPos; diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index bff10c3022..848eeee573 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -13,7 +13,7 @@ int32_t compareInt32Val(const void *pLeft, const void *pRight) { } int32_t compareInt64Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT64_VAL(pLeft) - GET_INT64_VAL(pRight); + int64_t ret = GET_INT64_VAL(pLeft) - GET_INT64_VAL(pRight); if (ret == 0) { return 0; } else { @@ -248,8 +248,9 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType) { case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: { - if (filterDataType == TSDB_DATA_TYPE_BIGINT) { + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + if (filterDataType == TSDB_DATA_TYPE_BIGINT || filterDataType == TSDB_DATA_TYPE_TIMESTAMP) { comparFn = compareInt64Val; break; } diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 138c75c197..beb831ea67 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -75,53 +75,6 @@ static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSk static SSkipListNode* tSkipListDoAppend(SSkipList *pSkipList, SSkipListNode *pNode); static SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); -//static __compar_fn_t getComparFunc(SSkipList *pSkipList, int32_t filterDataType) { -// __compar_fn_t comparFn = NULL; -// -// switch (pSkipList->keyInfo.type) { -// case TSDB_DATA_TYPE_TINYINT: -// case TSDB_DATA_TYPE_SMALLINT: -// case TSDB_DATA_TYPE_INT: -// case TSDB_DATA_TYPE_BIGINT: { -// if (filterDataType == TSDB_DATA_TYPE_BIGINT) { -// comparFn = compareInt64Val; -// break; -// } -// } -// case TSDB_DATA_TYPE_BOOL: { -// if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { -// comparFn = compareInt32Val; -// } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { -// comparFn = compareIntDoubleVal; -// } -// break; -// } -// case TSDB_DATA_TYPE_FLOAT: -// case TSDB_DATA_TYPE_DOUBLE: { -//// if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { -//// comparFn = compareDoubleIntVal; -//// } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { -//// comparFn = compareDoubleVal; -//// } -// if (filterDataType == TSDB_DATA_TYPE_DOUBLE) { -// comparFn = compareDoubleVal; -// } -// break; -// } -// case TSDB_DATA_TYPE_BINARY: -// comparFn = compareStrVal; -// break; -// case TSDB_DATA_TYPE_NCHAR: -// comparFn = compareWStrVal; -// break; -// default: -// comparFn = compareInt32Val; -// break; -// } -// -// return comparFn; -//} - static bool initForwardBackwardPtr(SSkipList* pSkipList) { uint32_t maxLevel = pSkipList->maxLevel; @@ -445,6 +398,11 @@ SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* iter->cur = forward[0]; // greater equals than the value } else { iter->cur = SL_GET_FORWARD_POINTER(forward[0], 0); + + if (ret == 0) { + assert(iter->cur != pSkipList->pTail); + iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); + } } return iter; diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index ee95199e5b..bf45c2d0af 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -26,7 +26,7 @@ #define EXTRA_BYTES 2 #define PRIMARY_TSCOL_REQUIRED(c) (((SColumnInfoData*)taosArrayGet(c, 0))->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) -#define QUERY_IS_ASC_QUERY(o) (o == TSDB_ORDER_ASC) +#define ASCENDING_ORDER_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) enum { @@ -154,6 +154,7 @@ tsdb_query_handle_t* tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond* pCond pQueryHandle->loadDataAfterSeek = false; pQueryHandle->isFirstSlot = true; + pQueryHandle->cur.fid = -1; size_t size = taosArrayGetSize(idList); assert(size >= 1); @@ -183,7 +184,7 @@ tsdb_query_handle_t* tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond* pCond * For ascending timestamp order query, query starts from data files. In contrast, buffer will be checked in the first place * in case of descending timestamp order query. */ - pQueryHandle->checkFiles = QUERY_IS_ASC_QUERY(pQueryHandle->order); + pQueryHandle->checkFiles = ASCENDING_ORDER_TRAVERSE(pQueryHandle->order); pQueryHandle->activeIndex = 0; // allocate buffer in order to load data blocks from file @@ -208,10 +209,11 @@ tsdb_query_handle_t* tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond* pCond static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); + pHandle->cur.fid = -1; + + STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - - STable* pTable = pTableCheckInfo->pTableObj; + STable* pTable = pCheckInfo->pTableObj; assert(pTable != NULL); // no data in cache, abort @@ -219,11 +221,10 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { return false; } - STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - pTable = pCheckInfo->pTableObj; - if (pCheckInfo->iter == NULL) { - pCheckInfo->iter = tSkipListCreateIter(pTable->mem->pData); + pCheckInfo->iter = tSkipListCreateIterFromVal(pTable->mem->pData, (const char*) &pCheckInfo->lastKey, + TSDB_DATA_TYPE_TIMESTAMP, pHandle->order); + if (pCheckInfo->iter == NULL) { return false; } @@ -240,9 +241,12 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { SDataRow row = SL_GET_NODE_DATA(node); pCheckInfo->lastKey = dataRowKey(row); // first timestamp in buffer + dTrace("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d", pHandle, + pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order); // all data in mem are checked already. - if (pTableCheckInfo->lastKey > pHandle->window.ekey) { + if ((pCheckInfo->lastKey > pHandle->window.ekey && ASCENDING_ORDER_TRAVERSE(pHandle->order)) || + (pCheckInfo->lastKey < pHandle->window.ekey && !ASCENDING_ORDER_TRAVERSE(pHandle->order))) { return false; } @@ -440,7 +444,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock SArray* sa = getDefaultLoadColumns(pQueryHandle, true); SQueryFilePos* cur = &pQueryHandle->cur; - if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { + if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) { // query ended in current block if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { @@ -611,11 +615,11 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf SDataCols* pCols = pCheckInfo->pDataCols; int32_t endPos = cur->pos; - if (QUERY_IS_ASC_QUERY(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { + if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { endPos = blockInfo.rows - 1; pQueryHandle->realNumOfRows = endPos - cur->pos + 1; pCheckInfo->lastKey = blockInfo.window.ekey + 1; - } else if (!QUERY_IS_ASC_QUERY(pQueryHandle->order) && pQueryHandle->window.ekey < blockInfo.window.skey) { + } else if (!ASCENDING_ORDER_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey < blockInfo.window.skey) { endPos = 0; pQueryHandle->realNumOfRows = cur->pos + 1; pCheckInfo->lastKey = blockInfo.window.ekey - 1; @@ -623,7 +627,7 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; endPos = vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfPoints, pQueryHandle->window.ekey, order); - if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { + if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) { if (endPos < cur->pos) { pQueryHandle->realNumOfRows = 0; return; @@ -940,7 +944,7 @@ static bool getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle) { int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { - int32_t type = QUERY_IS_ASC_QUERY(pQueryHandle->order)? QUERY_RANGE_GREATER_EQUAL:QUERY_RANGE_LESS_EQUAL; + int32_t type = ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)? QUERY_RANGE_GREATER_EQUAL:QUERY_RANGE_LESS_EQUAL; if (getFileCompInfo(pQueryHandle, &numOfBlocks, type) != TSDB_CODE_SUCCESS) { break; } @@ -968,7 +972,7 @@ static bool getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle) { return false; } - cur->slot = QUERY_IS_ASC_QUERY(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; + cur->slot = ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; cur->fid = pQueryHandle->pFileGroup->fileId; STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; @@ -993,16 +997,16 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { return getDataBlocksInFilesImpl(pQueryHandle); } else { - if ((cur->slot == pQueryHandle->numOfBlocks - 1 && QUERY_IS_ASC_QUERY(pQueryHandle->order)) || - (cur->slot == 0 && !QUERY_IS_ASC_QUERY(pQueryHandle->order))) { // all blocks + if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) || + (cur->slot == 0 && !ASCENDING_ORDER_TRAVERSE(pQueryHandle->order))) { // all blocks return getDataBlocksInFilesImpl(pQueryHandle); } else { // next block of the same file - int32_t step = QUERY_IS_ASC_QUERY(pQueryHandle->order)? 1:-1; + int32_t step = ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)? 1:-1; cur->slot += step; STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; - if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { + if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) { cur->pos = 0; } else { cur->pos = pBlockInfo->pBlock.compBlock->numOfPoints - 1; @@ -1035,7 +1039,7 @@ bool tsdbNextDataBlock(tsdb_query_handle_t* pqHandle) { size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); assert(numOfTables > 0); - if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { + if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) { if (pQueryHandle->checkFiles) { if (getDataBlocksInFiles(pQueryHandle)) { return true; @@ -1066,12 +1070,23 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int max int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns); *skey = INT64_MIN; - do/* (1) */{ + do { SSkipListNode* node = tSkipListIterGet(pIter); - if (node == NULL) break; + if (node == NULL) { + break; + } SDataRow row = SL_GET_NODE_DATA(node); - if (dataRowKey(row) > maxKey) break; + TSKEY key = dataRowKey(row); + + if ((key > maxKey && ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) || + (key < maxKey && !ASCENDING_ORDER_TRAVERSE(pQueryHandle->order))) { + + dTrace("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, + pQueryHandle->window.ekey); + + break; + } if (*skey == INT64_MIN) { *skey = dataRowKey(row); @@ -1080,9 +1095,18 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int max *ekey = dataRowKey(row); int32_t offset = 0; + char* pData = NULL; + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - memcpy(pColInfo->pData + numOfRows * pColInfo->info.bytes, dataRowTuple(row) + offset, pColInfo->info.bytes); + + if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)) { + pData = pColInfo->pData + numOfRows * pColInfo->info.bytes; + } else { + pData = pColInfo->pData + (maxRowsToRead - numOfRows - 1) * pColInfo->info.bytes; + } + + memcpy(pData, dataRowTuple(row) + offset, pColInfo->info.bytes); offset += pColInfo->info.bytes; } @@ -1093,6 +1117,18 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int max } while(tSkipListIterNext(pIter)); + assert(numOfRows <= maxRowsToRead); + + // if the buffer is not full in case of descending order query, move the data in the front of the buffer + if (!ASCENDING_ORDER_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { + int32_t emptySize = maxRowsToRead - numOfRows; + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); + } + } + return numOfRows; } @@ -1105,6 +1141,8 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t* pQueryHandle) { TSKEY skey = 0, ekey = 0; int32_t rows = 0; + int32_t step = ASCENDING_ORDER_TRAVERSE(pHandle->order)? 1:-1; + // data in file if (pHandle->cur.fid >= 0) { STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot]; @@ -1126,7 +1164,7 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t* pQueryHandle) { ekey = *(TSKEY*)((char*)pColInfoEx->pData + TSDB_KEYSIZE * (rows - 1)); // update the last key value - pBlockInfo->pTableCheckInfo->lastKey = ekey + 1; + pBlockInfo->pTableCheckInfo->lastKey = ekey + step; } } else { STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); @@ -1138,12 +1176,16 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t* pQueryHandle) { rows = tsdbReadRowsFromCache(pCheckInfo->iter, pHandle->window.ekey, 2, &skey, &ekey, pHandle); // update the last key value - pCheckInfo->lastKey = ekey + 1; + pCheckInfo->lastKey = ekey + step; } } SDataBlockInfo blockInfo = { - .uid = pTable->tableId.uid, .sid = pTable->tableId.tid, .rows = rows, .window = {.skey = skey, .ekey = ekey}}; + .uid = pTable->tableId.uid, + .sid = pTable->tableId.tid, + .rows = rows, + .window = {.skey = MIN(skey, ekey), .ekey = MAX(skey, ekey)} + }; return blockInfo; } @@ -1399,7 +1441,10 @@ int32_t tsdbQueryTags(tsdb_repo_t* tsdb, int64_t uid, const char* pTagCond, size void tsdbCleanupQueryHandle(tsdb_query_handle_t queryHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle; - + if (pQueryHandle == NULL) { + return; + } + size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); for (int32_t i = 0; i < size; ++i) { STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); From 5b3353138985a460872ada5b1b52d0142c4e9666 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 13 Apr 2020 23:14:25 +0800 Subject: [PATCH 76/78] [TD-52] refact sdb code --- src/mnode/inc/mgmtSdb.h | 8 +- src/mnode/src/mgmtDb.c | 10 +- src/mnode/src/mgmtSdb.c | 285 +++++++++++++++++++------------------ src/mnode/src/mgmtTable.c | 44 +++--- src/mnode/src/mgmtUser.c | 10 +- src/mnode/src/mgmtVgroup.c | 14 +- 6 files changed, 189 insertions(+), 182 deletions(-) diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index 95f3e6d39b..83afa2a081 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -33,13 +33,13 @@ typedef enum { } ESdbTable; typedef enum { - SDB_KEY_TYPE_STRING, - SDB_KEY_TYPE_AUTO + SDB_KEY_STRING, + SDB_KEY_AUTO } ESdbKeyType; typedef enum { - SDB_OPER_TYPE_GLOBAL, - SDB_OPER_TYPE_LOCAL + SDB_OPER_GLOBAL, + SDB_OPER_LOCAL } ESdbOperType; typedef struct { diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 2de9656e3e..946ec29d8c 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -116,7 +116,7 @@ int32_t mgmtInitDbs() { .hashSessions = TSDB_MAX_DBS, .maxRowSize = tsDbUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, - .keyType = SDB_KEY_TYPE_STRING, + .keyType = SDB_KEY_STRING, .insertFp = mgmtDbActionInsert, .deleteFp = mgmtDbActionDelete, .updateFp = mgmtDbActionUpdate, @@ -311,7 +311,7 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { pDb->cfg = *pCreate; SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsDbSdb, .pObj = pDb, .rowSize = sizeof(SDbObj) @@ -664,7 +664,7 @@ static int32_t mgmtSetDbDropping(SDbObj *pDb) { pDb->status = true; SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsDbSdb, .pObj = pDb, .rowSize = tsDbUpdateSize @@ -749,7 +749,7 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) { if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) { pDb->cfg = newCfg; SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsDbSdb, .pObj = pDb, .rowSize = tsDbUpdateSize @@ -807,7 +807,7 @@ static void mgmtDropDb(SQueuedMsg *pMsg) { mPrint("db:%s, drop db from sdb", pDb->name); SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsDbSdb, .pObj = pDb }; diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 6a229e716c..5202f8511c 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -31,8 +31,8 @@ typedef struct { int32_t code; int64_t version; - void * pSync; - void * pWal; + void * sync; + void * wal; sem_t sem; pthread_mutex_t mutex; } SSdbSync; @@ -99,13 +99,13 @@ static char *sdbGetActionStr(int32_t action) { static char *sdbGetkeyStr(SSdbTable *pTable, void *row) { static char str[16]; switch (pTable->keyType) { - case SDB_KEY_TYPE_STRING: + case SDB_KEY_STRING: return (char *)row; - case SDB_KEY_TYPE_AUTO: + case SDB_KEY_AUTO: sprintf(str, "%d", *(int32_t *)row); return str; default: - return "unknown"; + return "invalid"; } } @@ -135,14 +135,14 @@ int32_t sdbInit() { pthread_mutex_init(&tsSdbSync->mutex, NULL); SWalCfg walCfg = {.commitLog = 2, .wals = 2, .keep = 1}; - tsSdbSync->pWal = walOpen(tsMnodeDir, &walCfg); - if (tsSdbSync->pWal == NULL) { + tsSdbSync->wal = walOpen(tsMnodeDir, &walCfg); + if (tsSdbSync->wal == NULL) { sdbError("failed to open sdb in %s", tsMnodeDir); return -1; } sdbTrace("open sdb file for read"); - walRestore(tsSdbSync->pWal, tsSdbSync, sdbProcessWrite); + walRestore(tsSdbSync->wal, tsSdbSync, sdbProcessWrite); int32_t totalRows = 0; int32_t numOfTables = 0; @@ -166,7 +166,7 @@ void sdbCleanUp() { if (tsSdbSync) { sem_destroy(&tsSdbSync->sem); pthread_mutex_destroy(&tsSdbSync->mutex); - walClose(tsSdbSync->pWal); + walClose(tsSdbSync->wal); tsSdbSync = NULL; } } @@ -174,25 +174,28 @@ void sdbCleanUp() { void sdbIncRef(void *handle, void *pRow) { if (pRow) { SSdbTable *pTable = handle; - int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos); + int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); atomic_add_fetch_32(pRefCount, 1); - //if (0 && strcmp(pTable->tableName, "dnodes") == 0) { - sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); - //} + if (0 && strcmp(pTable->tableName, "dnodes") == 0) { + sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), + *pRefCount); + } } } void sdbDecRef(void *handle, void *pRow) { if (pRow) { SSdbTable *pTable = handle; - int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos); - int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); - //if (0 && strcmp(pTable->tableName, "dnodes") == 0) { - sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); - //} - int8_t* updateEnd = pRow + pTable->refCountPos - 1; + int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); + int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); + if (0 && strcmp(pTable->tableName, "dnodes") == 0) { + sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), + *pRefCount); + } + int8_t *updateEnd = pRow + pTable->refCountPos - 1; if (refCount <= 0 && *updateEnd) { - sdbTrace("table:%s, record:%s:%s:%d is destroyed", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); + sdbTrace("table:%s, record:%s:%s:%d is destroyed", pTable->tableName, pTable->tableName, + sdbGetkeyStr(pTable, pRow), *pRefCount); SSdbOperDesc oper = {.pObj = pRow}; (*pTable->destroyFp)(&oper); } @@ -228,7 +231,7 @@ void *sdbGetRow(void *handle, void *key) { return pMeta->row; } -static int32_t sdbInsertLocal(SSdbTable* pTable, SSdbOperDesc *pOper) { +static int32_t sdbInsertLocal(SSdbTable *pTable, SSdbOperDesc *pOper) { SRowMeta rowMeta; rowMeta.rowSize = pOper->rowSize; rowMeta.row = pOper->pObj; @@ -239,41 +242,144 @@ static int32_t sdbInsertLocal(SSdbTable* pTable, SSdbOperDesc *pOper) { pTable->numOfRows++; pthread_mutex_unlock(&pTable->mutex); - sdbTrace("table:%s, insert record:%s, numOfRows:%d", pTable->tableName, - sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows); + sdbTrace("table:%s, insert record:%s, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), + pTable->numOfRows); (*pTable->insertFp)(pOper); return TSDB_CODE_SUCCESS; } -static int32_t sdbDeleteLocal(SSdbTable* pTable, SSdbOperDesc *pOper) { +static int32_t sdbDeleteLocal(SSdbTable *pTable, SSdbOperDesc *pOper) { pthread_mutex_lock(&pTable->mutex); (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj); pTable->numOfRows--; pthread_mutex_unlock(&pTable->mutex); - sdbTrace("table:%s, delete record:%s, numOfRows:%d", pTable->tableName, - sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows); + sdbTrace("table:%s, delete record:%s, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), + pTable->numOfRows); (*pTable->deleteFp)(pOper); - int8_t* updateEnd = pOper->pObj + pTable->refCountPos - 1; + int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1; *updateEnd = 1; sdbDecRef(pTable, pOper->pObj); - + return TSDB_CODE_SUCCESS; } -static int32_t sdbUpdateLocal(SSdbTable* pTable, SSdbOperDesc *pOper) { - sdbTrace("table:%s, update record:%s, numOfRows:%d", pTable->tableName, - sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows); +static int32_t sdbUpdateLocal(SSdbTable *pTable, SSdbOperDesc *pOper) { + sdbTrace("table:%s, update record:%s, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), + pTable->numOfRows); (*pTable->updateFp)(pOper); return TSDB_CODE_SUCCESS; } +static int32_t sdbProcessWriteFromApp(SSdbTable *pTable, SWalHead *pHead, int32_t action) { + int32_t code = 0; + + pthread_mutex_lock(&tsSdbSync->mutex); + tsSdbSync->version++; + pHead->version = tsSdbSync->version; + + code = sdbForwardDbReqToPeer(pHead); + if (code != TSDB_CODE_SUCCESS) { + pthread_mutex_unlock(&tsSdbSync->mutex); + sdbError("table:%s, failed to forward %s record:%s from file, version:%" PRId64 ", reason:%s", pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code)); + return code; + } + + code = walWrite(tsSdbSync->wal, pHead); + pthread_mutex_unlock(&tsSdbSync->mutex); + + if (code < 0) { + sdbError("table:%s, failed to %s record:%s to file, version:%" PRId64 ", reason:%s", pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code)); + } else { + sdbTrace("table:%s, success to %s record:%s to file, version:%" PRId64, pTable->tableName, sdbGetActionStr(action), + sdbGetkeyStr(pTable, pHead->cont), pHead->version); + } + + walFsync(tsSdbSync->wal); + free(pHead); + + return code; +} + +static int32_t sdbProcessWriteFromWal(SSdbTable *pTable, SWalHead *pHead, int32_t action) { + pthread_mutex_lock(&tsSdbSync->mutex); + if (pHead->version <= tsSdbSync->version) { + pthread_mutex_unlock(&tsSdbSync->mutex); + return TSDB_CODE_SUCCESS; + } else if (pHead->version != tsSdbSync->version + 1) { + pthread_mutex_unlock(&tsSdbSync->mutex); + sdbError("table:%s, failed to restore %s record:%s from file, version:%" PRId64 " too large, sdb version:%" PRId64, + pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, + tsSdbSync->version); + return TSDB_CODE_OTHERS; + } + + tsSdbSync->version = pHead->version; + sdbTrace("table:%s, success to restore %s record:%s from file, version:%" PRId64, pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); + + int32_t code = -1; + if (action == SDB_ACTION_INSERT) { + SSdbOperDesc oper = { + .rowSize = pHead->len, + .rowData = pHead->cont, + .table = pTable, + }; + code = (*pTable->decodeFp)(&oper); + if (code < 0) { + sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); + pthread_mutex_unlock(&tsSdbSync->mutex); + return code; + } + + code = sdbInsertLocal(pTable, &oper); + } else if (action == SDB_ACTION_DELETE) { + SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont); + assert(rowMeta != NULL && rowMeta->row != NULL); + + SSdbOperDesc oper = { + .table = pTable, + .pObj = rowMeta->row, + }; + + code = sdbDeleteLocal(pTable, &oper); + } else if (action == SDB_ACTION_UPDATE) { + SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont); + assert(rowMeta != NULL && rowMeta->row != NULL); + + SSdbOperDesc oper1 = { + .table = pTable, + .pObj = rowMeta->row, + }; + sdbDeleteLocal(pTable, &oper1); + + SSdbOperDesc oper2 = { + .rowSize = pHead->len, + .rowData = pHead->cont, + .table = pTable, + }; + code = (*pTable->decodeFp)(&oper2); + if (code < 0) { + sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName, + sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); + pthread_mutex_unlock(&tsSdbSync->mutex); + return code; + } + code = sdbInsertLocal(pTable, &oper2); + } + + pthread_mutex_unlock(&tsSdbSync->mutex); + return code; +} + static int sdbProcessWrite(void *param, void *data, int type) { SWalHead *pHead = data; - int32_t code = 0; int32_t tableId = pHead->msgType / 10; int32_t action = pHead->msgType % 10; @@ -281,108 +387,9 @@ static int sdbProcessWrite(void *param, void *data, int type) { assert(pTable != NULL); if (pHead->version == 0) { - // from mgmt, update version - pthread_mutex_lock(&tsSdbSync->mutex); - tsSdbSync->version++; - pHead->version = tsSdbSync->version; - - code = sdbForwardDbReqToPeer(pHead); - if (code != TSDB_CODE_SUCCESS) { - pthread_mutex_unlock(&tsSdbSync->mutex); - sdbError("table:%s, failed to forward %s record:%s from file, version:%" PRId64 ", reason:%s", pTable->tableName, - sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code)); - return code; - } - - code = walWrite(tsSdbSync->pWal, pHead); - pthread_mutex_unlock(&tsSdbSync->mutex); - - if (code < 0) { - sdbError("table:%s, failed to %s record:%s to file, version:%" PRId64 ", reason:%s", pTable->tableName, - sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code)); - } else { - sdbTrace("table:%s, success to %s record:%s to file, version:%" PRId64, pTable->tableName, sdbGetActionStr(action), - sdbGetkeyStr(pTable, pHead->cont), pHead->version); - } - - walFsync(tsSdbSync->pWal); - free(pHead); - - return code; + return sdbProcessWriteFromApp(pTable, pHead, action); } else { - // for data from WAL or forward, version may be smaller - pthread_mutex_lock(&tsSdbSync->mutex); - - if (pHead->version <= tsSdbSync->version) { - pthread_mutex_unlock(&tsSdbSync->mutex); - return TSDB_CODE_SUCCESS; - } else if (pHead->version != tsSdbSync->version + 1) { - pthread_mutex_unlock(&tsSdbSync->mutex); - sdbError("table:%s, failed to restore %s record:%s from file, version:%" PRId64 " too large, sdb version:%" PRId64, - pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, - tsSdbSync->version); - return TSDB_CODE_OTHERS; - } else { - tsSdbSync->version = pHead->version; - sdbTrace("table:%s, success to restore %s record:%s from file, version:%" PRId64, pTable->tableName, - sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); - } - - - - code = -1; - if (action == SDB_ACTION_INSERT) { - SSdbOperDesc oper = { - .rowSize = pHead->len, - .rowData = pHead->cont, - .table = pTable, - }; - code = (*pTable->decodeFp)(&oper); - if (code < 0) { - sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName, - sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); - pthread_mutex_unlock(&tsSdbSync->mutex); - return code; - } - - code = sdbInsertLocal(pTable, &oper); - } else if (action == SDB_ACTION_DELETE) { - SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont); - assert(rowMeta != NULL && rowMeta->row != NULL); - - SSdbOperDesc oper = { - .table = pTable, - .pObj = rowMeta->row, - }; - - code = sdbDeleteLocal(pTable, &oper); - } else if (action == SDB_ACTION_UPDATE) { - SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont); - assert(rowMeta != NULL && rowMeta->row != NULL); - - SSdbOperDesc oper1 = { - .table = pTable, - .pObj = rowMeta->row, - }; - sdbDeleteLocal(pTable, &oper1); - - SSdbOperDesc oper2 = { - .rowSize = pHead->len, - .rowData = pHead->cont, - .table = pTable, - }; - code = (*pTable->decodeFp)(&oper2); - if (code < 0) { - sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName, - sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version); - pthread_mutex_unlock(&tsSdbSync->mutex); - return code; - } - code = sdbInsertLocal(pTable, &oper2); - } - - pthread_mutex_unlock(&tsSdbSync->mutex); - return code; + return sdbProcessWriteFromWal(pTable, pHead, action); } } @@ -396,7 +403,7 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { return TSDB_CODE_ALREADY_THERE; } - if (pTable->keyType == SDB_KEY_TYPE_AUTO) { + if (pTable->keyType == SDB_KEY_AUTO) { pthread_mutex_lock(&pTable->mutex); *((uint32_t *)pOper->pObj) = ++pTable->autoIndex; @@ -407,7 +414,7 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { pthread_mutex_unlock(&pTable->mutex); } - if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + if (pOper->type == SDB_OPER_GLOBAL) { int32_t size = sizeof(SWalHead) + pTable->maxRowSize; SWalHead *pHead = calloc(1, size); pHead->version = 0; @@ -439,13 +446,13 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { void * pMetaRow = pMeta->row; assert(pMetaRow != NULL); - if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + if (pOper->type == SDB_OPER_GLOBAL) { int32_t rowSize = 0; switch (pTable->keyType) { - case SDB_KEY_TYPE_STRING: + case SDB_KEY_STRING: rowSize = strlen((char *)pOper->pObj) + 1; break; - case SDB_KEY_TYPE_AUTO: + case SDB_KEY_AUTO: rowSize = sizeof(uint64_t); break; default: @@ -479,7 +486,7 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { void * pMetaRow = pMeta->row; assert(pMetaRow != NULL); - if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + if (pOper->type == SDB_OPER_GLOBAL) { int32_t size = sizeof(SWalHead) + pTable->maxRowSize; SWalHead *pHead = calloc(1, size); pHead->version = 0; diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 19b747b745..ef38702a96 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -240,7 +240,7 @@ static int32_t mgmtChildTableActionUpdateAll() { if (pDb == NULL) { mError("ctable:%s, failed to get db, discard it", pTable->info.tableId); SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_LOCAL; + desc.type = SDB_OPER_LOCAL; desc.pObj = pTable; desc.table = tsChildTableSdb; sdbDeleteRow(&desc); @@ -254,7 +254,7 @@ static int32_t mgmtChildTableActionUpdateAll() { mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->sid); pTable->vgId = 0; SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_LOCAL; + desc.type = SDB_OPER_LOCAL; desc.pObj = pTable; desc.table = tsChildTableSdb; sdbDeleteRow(&desc); @@ -268,7 +268,7 @@ static int32_t mgmtChildTableActionUpdateAll() { pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); pTable->vgId = 0; SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_LOCAL; + desc.type = SDB_OPER_LOCAL; desc.pObj = pTable; desc.table = tsChildTableSdb; sdbDeleteRow(&desc); @@ -280,7 +280,7 @@ static int32_t mgmtChildTableActionUpdateAll() { mError("ctable:%s, vgroup:%d tableList is null", pTable->info.tableId, pTable->vgId); pTable->vgId = 0; SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_LOCAL; + desc.type = SDB_OPER_LOCAL; desc.pObj = pTable; desc.table = tsChildTableSdb; sdbDeleteRow(&desc); @@ -294,7 +294,7 @@ static int32_t mgmtChildTableActionUpdateAll() { mError("ctable:%s, stable:%s not exist", pTable->info.tableId, pTable->superTableId); pTable->vgId = 0; SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_LOCAL; + desc.type = SDB_OPER_LOCAL; desc.pObj = pTable; desc.table = tsChildTableSdb; sdbDeleteRow(&desc); @@ -318,7 +318,7 @@ static int32_t mgmtInitChildTables() { .hashSessions = tsMaxTables, .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, - .keyType = SDB_KEY_TYPE_STRING, + .keyType = SDB_KEY_STRING, .insertFp = mgmtChildTableActionInsert, .deleteFp = mgmtChildTableActionDelete, .updateFp = mgmtChildTableActionUpdate, @@ -433,7 +433,7 @@ static int32_t mgmtInitSuperTables() { .hashSessions = TSDB_MAX_SUPER_TABLES, .maxRowSize = tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, - .keyType = SDB_KEY_TYPE_STRING, + .keyType = SDB_KEY_STRING, .insertFp = mgmtSuperTableActionInsert, .deleteFp = mgmtSuperTableActionDelete, .updateFp = mgmtSuperTableActionUpdate, @@ -676,7 +676,7 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) { } SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, .rowSize = sizeof(SSuperTableObj) + schemaSize @@ -700,7 +700,7 @@ static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) { mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS); } else { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable }; @@ -751,7 +751,7 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i pStable->sversion++; SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, .rowSize = tsSuperTableUpdateSize @@ -782,7 +782,7 @@ static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { pStable->schema = realloc(pStable->schema, schemaSize); SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, .rowSize = tsSuperTableUpdateSize @@ -817,7 +817,7 @@ static int32_t mgmtModifySuperTableTagName(SSuperTableObj *pStable, char *oldTag strncpy(schema->name, newTagName, TSDB_COL_NAME_LEN); SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, .rowSize = tsSuperTableUpdateSize @@ -876,7 +876,7 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc } SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, .rowSize = tsSuperTableUpdateSize @@ -913,7 +913,7 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch } SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, .rowSize = tsSuperTableUpdateSize @@ -1061,7 +1061,7 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_LOCAL, + .type = SDB_OPER_LOCAL, .table = tsSuperTableSdb, .pObj = pTable, }; @@ -1276,7 +1276,7 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj } SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_GLOBAL; + desc.type = SDB_OPER_GLOBAL; desc.pObj = pTable; desc.table = tsChildTableSdb; @@ -1430,7 +1430,7 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc } SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable, .rowSize = tsChildTableUpdateSize @@ -1464,7 +1464,7 @@ static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, ch } SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable, .rowSize = tsChildTableUpdateSize @@ -1606,7 +1606,7 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) { if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_LOCAL, + .type = SDB_OPER_LOCAL, .table = tsChildTableSdb, .pObj = pTable, }; @@ -1635,7 +1635,7 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { if (pTable->superTable == pStable) { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_LOCAL, + .type = SDB_OPER_LOCAL, .table = tsChildTableSdb, .pObj = pTable, }; @@ -1724,7 +1724,7 @@ static void mgmtProcessDropChildTableRsp(SRpcMsg *rpcMsg) { } SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable }; @@ -1767,7 +1767,7 @@ static void mgmtProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { queueMsg->thandle, tstrerror(rpcMsg->code)); SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable }; diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index b216e8f36d..3a49e56331 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -104,7 +104,7 @@ int32_t mgmtInitUsers() { .hashSessions = TSDB_MAX_USERS, .maxRowSize = tsUserUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, - .keyType = SDB_KEY_TYPE_STRING, + .keyType = SDB_KEY_STRING, .insertFp = mgmtUserActionInsert, .deleteFp = mgmtUserActionDelete, .updateFp = mgmtUserActionUpdate, @@ -144,7 +144,7 @@ void mgmtReleaseUser(SUserObj *pUser) { static int32_t mgmtUpdateUser(SUserObj *pUser) { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsUserSdb, .pObj = pUser, .rowSize = tsUserUpdateSize @@ -192,7 +192,7 @@ int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { } SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsUserSdb, .pObj = pUser, .rowSize = sizeof(SUserObj) @@ -209,7 +209,7 @@ int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { static int32_t mgmtDropUser(SUserObj *pUser) { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsUserSdb, .pObj = pUser }; @@ -483,7 +483,7 @@ void mgmtDropAllUsers(SAcctObj *pAcct) { if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_LOCAL, + .type = SDB_OPER_LOCAL, .table = tsUserSdb, .pObj = pUser, }; diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 09fdffb68e..19468dc547 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -166,7 +166,7 @@ int32_t mgmtInitVgroups() { .hashSessions = TSDB_MAX_VGROUPS, .maxRowSize = tsVgUpdateSize, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, - .keyType = SDB_KEY_TYPE_AUTO, + .keyType = SDB_KEY_AUTO, .insertFp = mgmtVgroupActionInsert, .deleteFp = mgmtVgroupActionDelete, .updateFp = mgmtVgroupActionUpdate, @@ -203,7 +203,7 @@ SVgObj *mgmtGetVgroup(int32_t vgId) { void mgmtUpdateVgroup(SVgObj *pVgroup) { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, .pObj = pVgroup, .rowSize = tsVgUpdateSize @@ -247,7 +247,7 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) { } SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, .pObj = pVgroup, .rowSize = sizeof(SVgObj) @@ -279,7 +279,7 @@ void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle) { mTrace("vgroup:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes); mgmtSendDropVgroupMsg(pVgroup, NULL); SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, .pObj = pVgroup }; @@ -586,7 +586,7 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { mgmtAddToShellQueue(newMsg); } else { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, .pObj = pVgroup }; @@ -649,7 +649,7 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) { if (queueMsg->received != queueMsg->expected) return; SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, + .type = SDB_OPER_GLOBAL, .table = tsVgroupSdb, .pObj = pVgroup }; @@ -706,7 +706,7 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) { if (strncmp(pDropDb->name, pVgroup->dbName, dbNameLen) == 0) { SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_LOCAL, + .type = SDB_OPER_LOCAL, .table = tsVgroupSdb, .pObj = pVgroup, }; From 1fb5a8c2fd8ff2819f7450924125f5a1795ad54a Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 14 Apr 2020 00:01:31 +0800 Subject: [PATCH 77/78] fix some memory lost --- src/dnode/src/dnodeMClient.c | 1 + src/mnode/src/mgmtSdb.c | 3 ++- src/plugins/http/inc/tgHandle.h | 1 + src/plugins/http/src/httpSystem.c | 4 +++- src/plugins/http/src/tgHandle.c | 5 +++++ 5 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/dnode/src/dnodeMClient.c b/src/dnode/src/dnodeMClient.c index f9794d040b..5dd015313a 100644 --- a/src/dnode/src/dnodeMClient.c +++ b/src/dnode/src/dnodeMClient.c @@ -249,6 +249,7 @@ static bool dnodeReadMnodeIpList() { PARSE_OVER: free(content); + cJSON_Delete(root); fclose(fp); return ret; } diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 5202f8511c..bc0d1b81f2 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -167,6 +167,7 @@ void sdbCleanUp() { sem_destroy(&tsSdbSync->sem); pthread_mutex_destroy(&tsSdbSync->mutex); walClose(tsSdbSync->wal); + free(tsSdbSync); tsSdbSync = NULL; } } @@ -576,5 +577,5 @@ void sdbCloseTable(void *handle) { pthread_mutex_destroy(&pTable->mutex); sdbTrace("table:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbNumOfTables); - tfree(pTable); + free(pTable); } diff --git a/src/plugins/http/inc/tgHandle.h b/src/plugins/http/inc/tgHandle.h index 5b8c49900f..5622694374 100644 --- a/src/plugins/http/inc/tgHandle.h +++ b/src/plugins/http/inc/tgHandle.h @@ -33,6 +33,7 @@ #define TG_PASS_URL_POS 3 void tgInitHandle(HttpServer *pServer); +void tgCleanupHandle(); bool tgProcessRquest(struct HttpContext *pContext); diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index aa66af9825..46f31a12d6 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -122,12 +122,14 @@ void httpStopSystem() { if (httpServer != NULL) { httpServer->online = false; } + tgCleanupHandle(); } void httpCleanUpSystem() { httpPrint("http service cleanup"); httpStopSystem(); -#if 0 + +#if 1 if (httpServer == NULL) { return; } diff --git a/src/plugins/http/src/tgHandle.c b/src/plugins/http/src/tgHandle.c index c22a7bbdef..2be39743bd 100644 --- a/src/plugins/http/src/tgHandle.c +++ b/src/plugins/http/src/tgHandle.c @@ -116,6 +116,7 @@ void tgFreeSchemas() { } free(tgSchemas.schemas); tgSchemas.size = 0; + tgSchemas.schemas = NULL; } } @@ -290,6 +291,10 @@ void tgInitHandle(HttpServer *pServer) { httpAddMethod(pServer, &tgDecodeMethod); } +void tgCleanupHandle() { + tgFreeSchemas(); +} + bool tgGetUserFromUrl(HttpContext *pContext) { HttpParser *pParser = &pContext->parser; if (pParser->path[TG_USER_URL_POS].len > TSDB_USER_LEN - 1 || pParser->path[TG_USER_URL_POS].len <= 0) { From 8863766f7322372f2b72534797ae8d80ede44399 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 14 Apr 2020 11:02:01 +0800 Subject: [PATCH 78/78] make print output be python2 and python3 compatible. --- tests/pytest/util/log.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/pytest/util/log.py b/tests/pytest/util/log.py index e570af85f5..c7032df3c4 100644 --- a/tests/pytest/util/log.py +++ b/tests/pytest/util/log.py @@ -15,6 +15,7 @@ import sys import os import time import datetime +from distutils.log import warn as printf class TDLog: @@ -22,27 +23,27 @@ class TDLog: self.path = "" def info(self, info): - print("%s %s" % (datetime.datetime.now(), info)) + printf("%s %s" % (datetime.datetime.now(), info)) def sleep(self, sec): - print("%s sleep %d seconds" % (datetime.datetime.now(), sec)) + printf("%s sleep %d seconds" % (datetime.datetime.now(), sec)) time.sleep(sec) def debug(self, err): - print("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err)) + printf("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err)) def success(self, info): - print("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info)) + printf("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info)) def notice(self, err): - print("\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err)) + printf("\033[1;33m%s %s\033[0m" % (datetime.datetime.now(), err)) def exit(self, err): - print("\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err)) + printf("\033[1;31m%s %s\033[0m" % (datetime.datetime.now(), err)) sys.exit(1) - def printNoPrefix(self, info): - print("\033[1;36m%s\033[0m" % (info)) + def printfNoPrefix(self, info): + printf("\033[1;36m%s\033[0m" % (info)) tdLog = TDLog()