From 7ccba2bdf2bb85eb7e5ddafea78e1c602e59efd3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 18 Jul 2020 15:12:08 +0800 Subject: [PATCH 01/10] [td-225] refactor codes and add some log. --- src/query/inc/qast.h | 1 - src/query/src/qExecutor.c | 106 +++++++++++++++++++------------------- src/query/src/qast.c | 4 +- src/tsdb/src/tsdbRead.c | 30 ++--------- 4 files changed, 59 insertions(+), 82 deletions(-) diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index 918604f8c9..00049b486d 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -45,7 +45,6 @@ typedef void (*__do_filter_suppl_fn_t)(void *, void *); * */ typedef struct tQueryInfo { - int32_t colIndex; // index of column in schema uint8_t optr; // expression operator SSchema sch; // schema of tags char* q; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6dba5cbd2a..42fc90f478 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -50,11 +50,6 @@ #define SDATA_BLOCK_INITIALIZER (SDataBlockInfo) {{0}, 0} -/* get the qinfo struct address from the query struct address */ -#define GET_COLUMN_BYTES(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].base.colInfo.colIndex].bytes) -#define GET_COLUMN_TYPE(query, colidx) ((query)->colList[(query)->pSelectExpr[colidx].base.colInfo.colIndex].type) - enum { // when query starts to execute, this status will set QUERY_NOT_COMPLETED = 0x1u, @@ -394,15 +389,15 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin if (pWindowResInfo->size >= pWindowResInfo->capacity) { int64_t newCap = pWindowResInfo->capacity * 1.5; char *t = realloc(pWindowResInfo->pResult, newCap * sizeof(SWindowResult)); - if (t != NULL) { - pWindowResInfo->pResult = (SWindowResult *)t; - - int32_t inc = newCap - pWindowResInfo->capacity; - memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * inc); - } else { - // todo + if (t == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } + pWindowResInfo->pResult = (SWindowResult *)t; + + int32_t inc = newCap - pWindowResInfo->capacity; + memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * inc); + for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); } @@ -1053,9 +1048,9 @@ static char *getGroupbyColumnData(SQuery *pQuery, int16_t *type, int16_t *bytes, *type = pQuery->colList[colIndex].type; *bytes = pQuery->colList[colIndex].bytes; /* - * the colIndex is acquired from the first meter of all qualified meters in this vnode during query prepare - * stage, the remain meter may not have the required column in cache actually. So, the validation of required - * column in cache with the corresponding meter schema is reinforced. + * the colIndex is acquired from the first tables of all qualified tables in this vnode during query prepare + * stage, the remain tables may not have the required column in cache actually. So, the validation of required + * column in cache with the corresponding schema is reinforced. */ int32_t numOfCols = taosArrayGetSize(pDataBlock); @@ -2364,6 +2359,18 @@ static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVa } } +static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId) { + assert(pTagColList != NULL && numOfTags > 0); + + for(int32_t i = 0; i < numOfTags; ++i) { + if (pTagColList[i].colId == colId) { + return &pTagColList[i]; + } + } + + return NULL; +} + void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { SQuery *pQuery = pRuntimeEnv->pQuery; SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); @@ -2372,16 +2379,10 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { if (pQuery->numOfOutput == 1 && pExprInfo->base.functionId == TSDB_FUNC_TS_COMP) { assert(pExprInfo->base.numOfParams == 1); - // todo refactor extract function. - int16_t type = -1, bytes = -1; - for(int32_t i = 0; i < pQuery->numOfTags; ++i) { - if (pQuery->tagColList[i].colId == pExprInfo->base.arg->argValue.i64) { - type = pQuery->tagColList[i].type; - bytes = pQuery->tagColList[i].bytes; - } - } + int16_t tagColId = pExprInfo->base.arg->argValue.i64; + SColumnInfo* pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); - doSetTagValueInParam(tsdb, pTable, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes); + doSetTagValueInParam(tsdb, pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes); } else { // set tag value, by which the results are aggregated. for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) { @@ -2399,20 +2400,14 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { // set the join tag for first column SSqlFuncMsg *pFuncMsg = &pExprInfo->base; - if ((pFuncMsg->functionId == TSDB_FUNC_TS || pFuncMsg->functionId == TSDB_FUNC_PRJ) && pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX && - pRuntimeEnv->pTSBuf != NULL) { + if ((pFuncMsg->functionId == TSDB_FUNC_TS || pFuncMsg->functionId == TSDB_FUNC_PRJ) && pRuntimeEnv->pTSBuf != NULL && + pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { assert(pFuncMsg->numOfParams == 1); - // todo refactor - int16_t type = -1, bytes = -1; - for(int32_t i = 0; i < pQuery->numOfTags; ++i) { - if (pQuery->tagColList[i].colId == pExprInfo->base.arg->argValue.i64) { - type = pQuery->tagColList[i].type; - bytes = pQuery->tagColList[i].bytes; - } - } + int16_t tagColId = pExprInfo->base.arg->argValue.i64; + SColumnInfo* pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); - doSetTagValueInParam(tsdb, pTable, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes); + doSetTagValueInParam(tsdb, pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes); qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%"PRId64, pQInfo, pExprInfo->base.arg->argValue.i64, pRuntimeEnv->pCtx[0].tag.i64Key) } @@ -4149,6 +4144,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) } else { pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); } + return terrno; } @@ -4174,10 +4170,10 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) { } int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bool isSTableQuery) { - int32_t code = TSDB_CODE_SUCCESS; - + qDebug("QInfo:%p start to init qhandle", pQInfo); SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + int32_t code = TSDB_CODE_SUCCESS; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; pQuery->precision = tsdbGetCfg(tsdb)->precision; @@ -4186,6 +4182,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo setScanLimitationByResultBuffer(pQuery); changeExecuteScanOrder(pQInfo, false); + code = setupQueryHandle(tsdb, pQInfo, isSTableQuery); if (code != TSDB_CODE_SUCCESS) { return code; @@ -5693,7 +5690,6 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { } } - static int compareTableIdInfo(const void* a, const void* b) { const STableIdInfo* x = (const STableIdInfo*)a; const STableIdInfo* y = (const STableIdInfo*)b; @@ -5926,6 +5922,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ pQuery->window.ekey, pQuery->order.order); setQueryStatus(pQuery, QUERY_COMPLETED); pQInfo->tableqinfoGroupInfo.numOfTables = 0; + sem_post(&pQInfo->dataReady); return TSDB_CODE_SUCCESS; } @@ -6136,16 +6133,17 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo int32_t code = TSDB_CODE_SUCCESS; - char * tagCond = NULL, *tbnameCond = NULL; - SArray * pTableIdList = NULL; - SSqlFuncMsg **pExprMsg = NULL; - SColIndex * pGroupColIndex = NULL; - SColumnInfo* pTagColumnInfo = NULL; - SExprInfo *pExprs = NULL; - SSqlGroupbyExpr *pGroupbyExpr = NULL; + char *tagCond = NULL; + char *tbnameCond = NULL; + SArray *pTableIdList = NULL; + SSqlFuncMsg **pExprMsg = NULL; + SExprInfo *pExprs = NULL; + SColIndex *pGroupColIndex = NULL; + SColumnInfo *pTagColumnInfo = NULL; + SSqlGroupbyExpr *pGroupbyExpr = NULL; - if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo)) != - TSDB_CODE_SUCCESS) { + code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo); + if (code != TSDB_CODE_SUCCESS) { goto _over; } @@ -6172,7 +6170,8 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo bool isSTableQuery = false; STableGroupInfo tableGroupInfo = {0}; - + int64_t st = taosGetTimestampUs(); + if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) { STableIdInfo *id = taosArrayGet(pTableIdList, 0); @@ -6182,7 +6181,6 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo } } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) { isSTableQuery = true; - // TODO: need a macro from TSDB to check if table is super table // also note there's possibility that only one table in the super table if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) { @@ -6193,11 +6191,12 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(pGroupColIndex->flag)) { numOfGroupByCols = 0; } - + + qDebug("qmsg:%p query stable, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); code = tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &tableGroupInfo, pGroupColIndex, numOfGroupByCols); if (code != TSDB_CODE_SUCCESS) { - qError("qmsg:%p failed to QueryStable, reason: %s", pQueryMsg, tstrerror(code)); + qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code)); goto _over; } } else { @@ -6208,6 +6207,9 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo qDebug("qmsg:%p query on %zu tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); } + + int64_t el = taosGetTimestampUs() - st; + qDebug("qmsg:%p tag filter completed, elapsed time:%"PRId64"us", pQueryMsg, el); } else { assert(0); } @@ -6247,7 +6249,7 @@ _over: *pQInfo = NULL; } - // if failed to add ref for all meters in this query, abort current query + // if failed to add ref for all tables in this query, abort current query return code; } diff --git a/src/query/src/qast.c b/src/query/src/qast.c index da4eb8f3ba..42f9f214f5 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -678,7 +678,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, tstr *name = (tstr*) tsdbGetTableName(*(void**) pData); // todo speed up by using hash - if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { if (pQueryInfo->optr == TSDB_RELATION_IN) { addToResult = pQueryInfo->compare(name, pQueryInfo->q); } else if (pQueryInfo->optr == TSDB_RELATION_LIKE) { @@ -716,7 +716,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S } tQueryInfo *pQueryInfo = pExpr->_node.info; - if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) { + if (pQueryInfo->sch.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX && pQueryInfo->optr != TSDB_RELATION_LIKE) { tQueryIndexColumn(pSkipList, pQueryInfo, result); } else { tQueryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index eb34805de4..8463e791db 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -240,7 +240,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); - tsdbDebug("%p total numOfTable:%zu in query", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo)); + tsdbDebug("%p total numOfTable:%zu in query, %p", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qinfo); tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); @@ -1869,7 +1869,6 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta pHandle->statis[i].numOfNull = pBlockInfo->compBlock->numOfRows; } - // todo opt perf SColumnInfo* pColInfo = taosArrayGet(pHandle->pColumns, i); if (pColInfo->type == TSDB_DATA_TYPE_TIMESTAMP) { pHandle->statis[i].min = pBlockInfo->compBlock->keyFirst; @@ -1961,43 +1960,20 @@ static void destroyHelper(void* param) { free(param); } -#define TAG_INVALID_COLUMN_INDEX -2 -static int32_t getTagColumnIndex(STSchema* pTSchema, SSchema* pSchema) { - // filter on table name(TBNAME) - if (strcasecmp(pSchema->name, TSQL_TBNAME_L) == 0) { - return TSDB_TBNAME_COLUMN_INDEX; - } - - for(int32_t i = 0; i < schemaNCols(pTSchema); ++i) { - STColumn* pColumn = &pTSchema->columns[i]; - if (pColumn->bytes == pSchema->bytes && pColumn->type == pSchema->type && pColumn->colId == pSchema->colId) { - return i; - } - } - - return -2; -} - void filterPrepare(void* expr, void* param) { tExprNode* pExpr = (tExprNode*)expr; if (pExpr->_node.info != NULL) { return; } - int32_t i = 0; pExpr->_node.info = calloc(1, sizeof(tQueryInfo)); - STSchema* pTSSchema = (STSchema*) param; - + STSchema* pTSSchema = (STSchema*) param; tQueryInfo* pInfo = pExpr->_node.info; tVariant* pCond = pExpr->_node.pRight->pVal; SSchema* pSchema = pExpr->_node.pLeft->pSchema; - int32_t index = getTagColumnIndex(pTSSchema, pSchema); - assert((index >= 0 && i < TSDB_MAX_TAGS) || (index == TSDB_TBNAME_COLUMN_INDEX) || index == TAG_INVALID_COLUMN_INDEX); - pInfo->sch = *pSchema; - pInfo->colIndex = index; pInfo->optr = pExpr->_node.optr; pInfo->compare = getComparFunc(pSchema->type, pInfo->optr); pInfo->param = pTSSchema; @@ -2143,7 +2119,7 @@ bool indexedNodeFilterFp(const void* pNode, void* param) { char* val = NULL; - if (pInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (pInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { val = (char*) TABLE_NAME(pTable); } else { val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId); From 9bd890775f9186db34a8675da6e5712842965c18 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Sat, 18 Jul 2020 15:24:33 +0800 Subject: [PATCH 02/10] fix td-956 --- src/cq/src/cqMain.c | 6 +++--- src/tsdb/src/tsdbMain.c | 3 ++- src/vnode/src/vnodeMain.c | 14 +++++++------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 51cd471a6b..e0f30166c4 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -103,9 +103,6 @@ void cqClose(void *handle) { SCqContext *pContext = handle; if (handle == NULL) return; - taosTmrCleanUp(pContext->tmrCtrl); - pContext->tmrCtrl = NULL; - // stop all CQs cqStop(pContext); @@ -125,6 +122,9 @@ void cqClose(void *handle) { pthread_mutex_destroy(&pContext->mutex); + taosTmrCleanUp(pContext->tmrCtrl); + pContext->tmrCtrl = NULL; + cTrace("vgId:%d, CQ is closed", pContext->vgId); free(pContext); } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index ffaab375a3..e30164592d 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -148,7 +148,7 @@ void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { STsdbRepo *pRepo = (STsdbRepo *)repo; int vgId = REPO_ID(pRepo); - tsdbStopStream(repo); + tsdbStopStream(pRepo); if (toCommit) { tsdbAsyncCommit(pRepo); @@ -1126,6 +1126,7 @@ static void tsdbStartStream(STsdbRepo *pRepo) { } } + static void tsdbStopStream(STsdbRepo *pRepo) { STsdbMeta *pMeta = pRepo->tsdbMeta; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 98882e4c3c..249fb428e7 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -340,6 +340,13 @@ void vnodeRelease(void *pVnodeRaw) { tsdbCloseRepo(pVnode->tsdb, 1); pVnode->tsdb = NULL; + // stop continuous query + if (pVnode->cq) { + void *cq = pVnode->cq; + pVnode->cq = NULL; + cqClose(cq); + } + if (pVnode->wal) walClose(pVnode->wal); pVnode->wal = NULL; @@ -511,13 +518,6 @@ static void vnodeCleanUp(SVnodeObj *pVnode) { syncStop(sync); } - // stop continuous query - if (pVnode->cq) { - void *cq = pVnode->cq; - pVnode->cq = NULL; - cqClose(cq); - } - vTrace("vgId:%d, vnode will cleanup, refCount:%d", pVnode->vgId, pVnode->refCount); // release local resources only after cutting off outside connections From 16049c8720d46a7f4a3bb1eed2f3f67c4035dde2 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 18 Jul 2020 16:59:58 +0800 Subject: [PATCH 03/10] support query clients. --- tests/comparisonTest/tdengine/q5.txt | 1 + tests/comparisonTest/tdengine/tdengineTest.c | 75 ++++++++++++++------ 2 files changed, 55 insertions(+), 21 deletions(-) create mode 100644 tests/comparisonTest/tdengine/q5.txt diff --git a/tests/comparisonTest/tdengine/q5.txt b/tests/comparisonTest/tdengine/q5.txt new file mode 100644 index 0000000000..5f36955dd6 --- /dev/null +++ b/tests/comparisonTest/tdengine/q5.txt @@ -0,0 +1 @@ +select * from db.devices; diff --git a/tests/comparisonTest/tdengine/tdengineTest.c b/tests/comparisonTest/tdengine/tdengineTest.c index de92526337..3d78a3d0a5 100644 --- a/tests/comparisonTest/tdengine/tdengineTest.c +++ b/tests/comparisonTest/tdengine/tdengineTest.c @@ -13,8 +13,9 @@ typedef struct { char sql[256]; char dataDir[256]; int filesNum; - int writeClients; + int clients; int rowsPerRequest; + int write; } ProArgs; typedef struct { @@ -41,7 +42,7 @@ int main(int argc, char *argv[]) { statis.totalRows = 0; parseArg(argc, argv); - if (arguments.writeClients > 0) { + if (arguments.write) { writeData(); } else { readData(); @@ -52,7 +53,7 @@ void parseArg(int argc, char *argv[]) { strcpy(arguments.sql, "./sqlCmd.txt"); strcpy(arguments.dataDir, "./testdata"); arguments.filesNum = 2; - arguments.writeClients = 0; + arguments.clients = 1; arguments.rowsPerRequest = 100; for (int i = 1; i < argc; ++i) { @@ -83,12 +84,12 @@ void parseArg(int argc, char *argv[]) { exit(EXIT_FAILURE); } } - else if (strcmp(argv[i], "-writeClients") == 0) { + else if (strcmp(argv[i], "-clients") == 0) { if (i < argc - 1) { - arguments.writeClients = atoi(argv[++i]); + arguments.clients = atoi(argv[++i]); } else { - fprintf(stderr, "'-writeClients' requires a parameter, default:%d\n", arguments.writeClients); + fprintf(stderr, "'-clients' requires a parameter, default:%d\n", arguments.clients); exit(EXIT_FAILURE); } } @@ -101,6 +102,9 @@ void parseArg(int argc, char *argv[]) { exit(EXIT_FAILURE); } } + else if (strcmp(argv[i], "-w") == 0) { + arguments.write = 1; + } } } @@ -215,7 +219,7 @@ void writeDataImp(void *param) { void writeData() { printf("write data\n"); - printf("---- writeClients: %d\n", arguments.writeClients); + printf("---- clients: %d\n", arguments.clients); printf("---- dataDir: %s\n", arguments.dataDir); printf("---- numOfFiles: %d\n", arguments.filesNum); printf("---- rowsPerRequest: %d\n", arguments.rowsPerRequest); @@ -243,12 +247,12 @@ void writeData() { int64_t st = getTimeStampMs(); - int a = arguments.filesNum / arguments.writeClients; - int b = arguments.filesNum % arguments.writeClients; + int a = arguments.filesNum / arguments.clients; + int b = arguments.filesNum % arguments.clients; int last = 0; - ThreadObj *threads = calloc((size_t)arguments.writeClients, sizeof(ThreadObj)); - for (int i = 0; i < arguments.writeClients; ++i) { + ThreadObj *threads = calloc((size_t)arguments.clients, sizeof(ThreadObj)); + for (int i = 0; i < arguments.clients; ++i) { ThreadObj *pthread = threads + i; pthread_attr_t thattr; pthread->threadId = i + 1; @@ -264,7 +268,7 @@ void writeData() { pthread_create(&pthread->pid, &thattr, (void *(*)(void *))writeDataImp, pthread); } - for (int i = 0; i < arguments.writeClients; i++) { + for (int i = 0; i < arguments.clients; i++) { pthread_join(threads[i].pid, NULL); } @@ -272,17 +276,15 @@ void writeData() { float seconds = (float)elapsed / 1000; float rs = (float)statis.totalRows / seconds; + free(threads); + printf("---- Spent %f seconds to insert %ld records, speed: %f Rows/Second\n", seconds, statis.totalRows, rs); } -void readData() { - printf("read data\n"); - printf("---- sql: %s\n", arguments.sql); - - void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0); - if (taos == NULL) - taos_error(taos); - +void readDataImp(void *param) +{ + ThreadObj *pThread = (ThreadObj *)param; + printf("Thread %d\n", pThread->threadId); FILE *fp = fopen(arguments.sql, "r"); if (fp == NULL) { printf("failed to open file %s\n", arguments.sql); @@ -290,6 +292,10 @@ void readData() { } printf("open file %s success\n", arguments.sql); + void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0); + if (taos == NULL) + taos_error(taos); + char *line = NULL; size_t len = 0; while (!feof(fp)) { @@ -325,9 +331,36 @@ void readData() { int64_t elapsed = getTimeStampMs() - st; float seconds = (float)elapsed / 1000; - printf("---- Spent %f seconds to query: %s", seconds, line); + printf("---- Spent %f seconds to retrieve %d records, Thread:%d query: %s\n", seconds, rows, pThread->threadId, line); } fclose(fp); } +void readData() { + printf("read data\n"); + printf("---- sql: %s\n", arguments.sql); + printf("---- clients: %d\n", arguments.clients); + + void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0); + if (taos == NULL) + taos_error(taos); + + ThreadObj *threads = calloc((size_t)arguments.clients, sizeof(ThreadObj)); + + for (int i = 0; i < arguments.clients; ++i) { + ThreadObj *pthread = threads + i; + pthread_attr_t thattr; + pthread->threadId = i + 1; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + pthread_create(&pthread->pid, &thattr, (void *(*)(void *))readDataImp, pthread); + } + + for (int i = 0; i < arguments.clients; i++) { + pthread_join(threads[i].pid, NULL); + } + + free(threads); +} + From db94a424cc3f2373c99ba0c970eb95ee9b4e361c Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 18 Jul 2020 17:02:15 +0800 Subject: [PATCH 04/10] modify argument of write loop script. --- .../perftest-scripts/tdengineTestWriteLoop.sh | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tests/perftest-scripts/tdengineTestWriteLoop.sh b/tests/perftest-scripts/tdengineTestWriteLoop.sh index 9f54bb6fc3..4cbb791678 100755 --- a/tests/perftest-scripts/tdengineTestWriteLoop.sh +++ b/tests/perftest-scripts/tdengineTestWriteLoop.sh @@ -5,7 +5,6 @@ NUM_LOOP=5 NUM_OF_FILES=100 rowsPerRequest=(1 100 500 1000 2000) -numOfClients=(1 2 3 4 5 6 7) function printTo { if $verbose ; then @@ -15,7 +14,7 @@ function printTo { function runTest { printf "R/R, " - for c in ${numOfClients[@]}; do + for c in `seq 1 $clients`; do if [ "$c" == "1" ]; then printf "$c client, " else @@ -26,7 +25,7 @@ function runTest { for r in ${rowsPerRequest[@]}; do printf "$r, " - for c in ${numOfClients[@]}; do + for c in `seq 1 $clients`; do totalRPR=0 for i in `seq 1 $NUM_LOOP`; do restartTaosd @@ -34,12 +33,12 @@ function runTest { printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ -dataDir $DATA_DIR \ -numOfFiles $NUM_OF_FILES \ - -writeClients $c \ + -w -clients $c \ -rowsPerRequest $r" RPR=`$TDTEST_DIR/tdengineTest \ -dataDir $DATA_DIR \ -numOfFiles 1 \ - -writeClients $c \ + -w -clients $c \ -rowsPerRequest $r \ | grep speed | awk '{print $(NF-1)}'` totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc` @@ -73,25 +72,29 @@ function restartTaosd { master=false develop=true verbose=false +clients=1 -for arg in "$@" -do - case $arg in +while : ; do + case $1 in -v) verbose=true - ;; + shift ;; master) master=true develop=false - ;; + shift ;; develop) master=false develop=true - ;; + shift ;; + + -c) + clients=$2 + shift 2;; *) - ;; + break ;; esac done From 49f8c9d822e3452289de7dec416f585bddec7725 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 18 Jul 2020 17:03:32 +0800 Subject: [PATCH 05/10] add q5 query script. --- tests/perftest-scripts/tdengineTestQ5Loop.sh | 107 +++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100755 tests/perftest-scripts/tdengineTestQ5Loop.sh diff --git a/tests/perftest-scripts/tdengineTestQ5Loop.sh b/tests/perftest-scripts/tdengineTestQ5Loop.sh new file mode 100755 index 0000000000..9eff74f563 --- /dev/null +++ b/tests/perftest-scripts/tdengineTestQ5Loop.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +DATA_DIR=/mnt/root/testdata +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +TDTESTQ5OUT=tdengineTestQ5.out + +function runTest { + totalThroughput=0 + for i in `seq 1 $NUM_LOOP`; do + for c in `seq 1 $clients`; do + records[$c]=0 + spentTime[$c]=0 + throughput[$c]=0 + done + printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ + -clients $clients -sql q5.txt" + restartTaosd + beginMS=`date +%s%3N` + $TDTEST_DIR/tdengineTest \ + -clients $clients -sql $TDTEST_DIR/q5.txt > $TDTESTQ5OUT + endMS=`date +%s%3N` + totalRecords=0 + for c in `seq 1 $clients`; do + records[$c]=`grep Thread:$c $TDTESTQ5OUT | awk '{print $7}'` + totalRecords=`echo "$totalRecords + ${records[$c]}"|bc` + done + spending=`echo "scale=4; x = ($endMS - $beginMS)/1000; if (x<1) print 0; x"|bc` + throughput=`echo "scale=4; x= $totalRecords / $spending; if (x<1) print 0; x" | bc` + printTo "spending: $spending sec, throughput: $throughput" + totalThroughput=`echo "scale=4; x = $totalThroughput + $throughput; if(x<1) print 0; x"|bc` + done + avgThrougput=`echo "scale=4; x = $totalThroughput / $NUM_LOOP; if (x<1) print 0; x"|bc` + echo "avg Throughput: $avgThrougput" +} + +function restartTaosd { + printTo "Stop taosd" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + + printTo "Start taosd" + $TAOSD_DIR/taosd > /dev/null 2>&1 & + sleep 10 +} + +################ Main ################ + +master=false +develop=true +verbose=false + +clients=1 + +while : ; do + case $1 in + -v) + verbose=true + shift ;; + + master) + master=true + develop=false + shift ;; + + develop) + master=false + develop=true + shift ;; + + -c) + clients=$2 + shift 2;; + + *) + break ;; + esac +done + +if $master ; then + printTo "Test master branch.." + cp /mnt/root/cfg/master/taos.cfg /etc/taos/taos.cfg + WORK_DIR=/mnt/root/TDengine.master +else + printTo "Test develop branch.." + cp /mnt/root/cfg/10billion/taos.cfg /etc/taos/taos.cfg + WORK_DIR=/mnt/root/TDengine +fi + +TAOSD_DIR=$WORK_DIR/debug/build/bin +TDTEST_DIR=$WORK_DIR/tests/comparisonTest/tdengine + +runTest + +printTo "Test done!" From fdfa789cbcffb9f621badec2d09e6c8ca3ebda10 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 18 Jul 2020 12:13:01 +0000 Subject: [PATCH 06/10] replace ipList and ipSet to epSet --- src/client/inc/tscUtil.h | 2 +- src/client/inc/tsclient.h | 6 +- src/client/src/tscSQLParser.c | 4 +- src/client/src/tscServer.c | 102 ++++++++++----------- src/client/src/tscSql.c | 4 +- src/client/src/tscSubquery.c | 14 +-- src/client/src/tscSystem.c | 6 +- src/client/src/tscUtil.c | 16 ++-- src/dnode/inc/dnodeMgmt.h | 4 +- src/dnode/src/dnodeMgmt.c | 104 +++++++++++----------- src/dnode/src/dnodePeer.c | 24 ++--- src/dnode/src/dnodeShell.c | 4 +- src/inc/dnode.h | 6 +- src/inc/taosmsg.h | 10 +-- src/inc/trpc.h | 14 +-- src/kit/taosmigrate/taosmigrate.c | 8 +- src/kit/taosmigrate/taosmigrate.h | 2 +- src/kit/taosmigrate/taosmigrateDnodeCfg.c | 36 ++++---- src/mnode/inc/mnodeMnode.h | 6 +- src/mnode/inc/mnodeVgroup.h | 6 +- src/mnode/src/mnodeDnode.c | 10 +-- src/mnode/src/mnodeMnode.c | 48 +++++----- src/mnode/src/mnodePeer.c | 14 +-- src/mnode/src/mnodeRead.c | 14 +-- src/mnode/src/mnodeSdb.c | 2 +- src/mnode/src/mnodeShow.c | 4 +- src/mnode/src/mnodeTable.c | 28 +++--- src/mnode/src/mnodeVgroup.c | 58 ++++++------ src/mnode/src/mnodeWrite.c | 16 ++-- src/rpc/src/rpcMain.c | 60 ++++++------- src/rpc/test/rclient.c | 30 +++---- src/rpc/test/rsclient.c | 26 +++--- src/rpc/test/rserver.c | 2 +- 33 files changed, 345 insertions(+), 345 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 6a5c2d0099..786133a8f3 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -268,7 +268,7 @@ bool hasMoreClauseToTry(SSqlObj* pSql); void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows); void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp); -int tscSetMgmtIpListFromCfg(const char *first, const char *second); +int tscSetMgmtEpSetFromCfg(const char *first, const char *second); void* malloc_throw(size_t size); void* calloc_throw(size_t nmemb, size_t size); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 1b30c4ffca..ef8184990d 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -306,7 +306,7 @@ typedef struct SSqlObj { char * sqlstr; char retry; char maxRetry; - SRpcIpSet ipList; + SRpcEpSet epSet; char listed; tsem_t rspSem; SSqlCmd cmd; @@ -350,7 +350,7 @@ void tscInitMsgsFp(); int tsParseSql(SSqlObj *pSql, bool initial); -void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet); +void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet); int tscProcessSql(SSqlObj *pSql); int tscRenewTableMeta(SSqlObj *pSql, char *tableId); @@ -456,7 +456,7 @@ extern void * tscQhandle; extern int tscKeepConn[]; extern int tsInsertHeadSize; extern int tscNumOfThreads; -extern SRpcIpSet tscMgmtIpSet; +extern SRpcEpSet tscMgmtEpSet; extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index f9b12f3f6d..b97e486449 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2355,9 +2355,9 @@ bool validateIpAddress(const char* ip, size_t size) { strncpy(tmp, ip, size); - in_addr_t ipAddr = inet_addr(tmp); + in_addr_t epAddr = inet_addr(tmp); - return ipAddr != INADDR_NONE; + return epAddr != INADDR_NONE; } int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 99ac2249e0..398dea9d09 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -29,8 +29,8 @@ #define TSC_MGMT_VNODE 999 -SRpcIpSet tscMgmtIpSet; -SRpcIpSet tscDnodeIpSet; +SRpcEpSet tscMgmtEpSet; +SRpcEpSet tscDnodeEpSet; int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; @@ -44,44 +44,44 @@ void tscSaveSubscriptionProgress(void* sub); static int32_t minMsgSize() { return tsRpcHeadSize + 100; } -static void tscSetDnodeIpList(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { - SRpcIpSet* pIpList = &pSql->ipList; - pIpList->inUse = 0; +static void tscSetDnodeEpSet(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { + SRpcEpSet* pEpSet = &pSql->epSet; + pEpSet->inUse = 0; if (pVgroupInfo == NULL) { - pIpList->numOfIps = 0; + pEpSet->numOfEps = 0; return; } - pIpList->numOfIps = pVgroupInfo->numOfIps; - for(int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) { - strcpy(pIpList->fqdn[i], pVgroupInfo->ipAddr[i].fqdn); - pIpList->port[i] = pVgroupInfo->ipAddr[i].port; + pEpSet->numOfEps = pVgroupInfo->numOfEps; + for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { + strcpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn); + pEpSet->port[i] = pVgroupInfo->epAddr[i].port; } } -void tscPrintMgmtIp() { - if (tscMgmtIpSet.numOfIps <= 0) { - tscError("invalid mnode IP list:%d", tscMgmtIpSet.numOfIps); +void tscPrintMgmtEp() { + if (tscMgmtEpSet.numOfEps <= 0) { + tscError("invalid mnode EP list:%d", tscMgmtEpSet.numOfEps); } else { - for (int i = 0; i < tscMgmtIpSet.numOfIps; ++i) { - tscDebug("mnode index:%d %s:%d", i, tscMgmtIpSet.fqdn[i], tscMgmtIpSet.port[i]); + for (int i = 0; i < tscMgmtEpSet.numOfEps; ++i) { + tscDebug("mnode index:%d %s:%d", i, tscMgmtEpSet.fqdn[i], tscMgmtEpSet.port[i]); } } } -void tscSetMgmtIpList(SRpcIpSet *pIpList) { - tscMgmtIpSet.numOfIps = pIpList->numOfIps; - tscMgmtIpSet.inUse = pIpList->inUse; - for (int32_t i = 0; i < tscMgmtIpSet.numOfIps; ++i) { - tscMgmtIpSet.port[i] = htons(pIpList->port[i]); +void tscSetMgmtEpSet(SRpcEpSet *pEpSet) { + tscMgmtEpSet.numOfEps = pEpSet->numOfEps; + tscMgmtEpSet.inUse = pEpSet->inUse; + for (int32_t i = 0; i < tscMgmtEpSet.numOfEps; ++i) { + tscMgmtEpSet.port[i] = htons(pEpSet->port[i]); } } -void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) { - tscMgmtIpSet = *pIpSet; - tscDebug("mnode IP list is changed for ufp is called, numOfIps:%d inUse:%d", tscMgmtIpSet.numOfIps, tscMgmtIpSet.inUse); - for (int32_t i = 0; i < tscMgmtIpSet.numOfIps; ++i) { - tscDebug("index:%d fqdn:%s port:%d", i, tscMgmtIpSet.fqdn[i], tscMgmtIpSet.port[i]); +void tscUpdateEpSet(void *ahandle, SRpcEpSet *pEpSet) { + tscMgmtEpSet = *pEpSet; + tscDebug("mnode EP list is changed for ufp is called, numOfEps:%d inUse:%d", tscMgmtEpSet.numOfEps, tscMgmtEpSet.inUse); + for (int32_t i = 0; i < tscMgmtEpSet.numOfEps; ++i) { + tscDebug("index:%d fqdn:%s port:%d", i, tscMgmtEpSet.fqdn[i], tscMgmtEpSet.port[i]); } } @@ -95,7 +95,7 @@ void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) { UNUSED_FUNC static int32_t tscGetMgmtConnMaxRetryTimes() { int32_t factor = 2; - return tscMgmtIpSet.numOfIps * factor; + return tscMgmtEpSet.numOfEps * factor; } void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { @@ -111,9 +111,9 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { if (code == 0) { SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp; - SRpcIpSet * pIpList = &pRsp->ipList; - if (pIpList->numOfIps > 0) - tscSetMgmtIpList(pIpList); + SRpcEpSet * pEpSet = &pRsp->epSet; + if (pEpSet->numOfEps > 0) + tscSetMgmtEpSet(pEpSet); pSql->pTscObj->connId = htonl(pRsp->connId); @@ -185,7 +185,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { // set the mgmt ip list if (pSql->cmd.command >= TSDB_SQL_MGMT) { - pSql->ipList = tscMgmtIpSet; + pSql->epSet = tscMgmtEpSet; } memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); @@ -203,11 +203,11 @@ int tscSendMsgToServer(SSqlObj *pSql) { // Otherwise, the pSql object may have been released already during the response function, which is // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely // cause crash. - rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg); + rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); return TSDB_CODE_SUCCESS; } -void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { +void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { SSqlObj *pSql = (SSqlObj *)rpcMsg->ahandle; if (pSql == NULL || pSql->signature != pSql) { tscError("%p sql is already released", pSql); @@ -237,9 +237,9 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { } if (pCmd->command < TSDB_SQL_MGMT) { - if (pIpSet) pSql->ipList = *pIpSet; + if (pEpSet) pSql->epSet = *pEpSet; } else { - if (pIpSet) tscMgmtIpSet = *pIpSet; + if (pEpSet) tscMgmtEpSet = *pEpSet; } if (rpcMsg->pCont == NULL) { @@ -421,7 +421,7 @@ int tscProcessSql(SSqlObj *pSql) { return pSql->res.code; } } else if (pCmd->command < TSDB_SQL_LOCAL) { - pSql->ipList = tscMgmtIpSet; + pSql->epSet = tscMgmtEpSet; } else { // local handler return (*tscProcessMsgRsp[pCmd->command])(pSql); } @@ -525,10 +525,10 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // pSql->cmd.payloadLen is set during copying data into payload pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - tscSetDnodeIpList(pSql, &pTableMeta->vgroupInfo); + tscSetDnodeEpSet(pSql, &pTableMeta->vgroupInfo); - tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfIP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit, - pSql->ipList.numOfIps); + tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfEP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit, + pSql->epSet.numOfEps); return TSDB_CODE_SUCCESS; } @@ -568,7 +568,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char pVgroupInfo = &pTableMeta->vgroupInfo; } - tscSetDnodeIpList(pSql, pVgroupInfo); + tscSetDnodeEpSet(pSql, pVgroupInfo); if (pVgroupInfo != NULL) { pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); } @@ -580,7 +580,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char pQueryMsg->numOfTables = htonl(1); // set the number of tables pMsg += sizeof(STableIdInfo); - } else { // it is a subquery of the super table query, this IP info is acquired from vgroupInfo + } else { // it is a subquery of the super table query, this EP info is acquired from vgroupInfo int32_t index = pTableMetaInfo->vgroupIndex; int32_t numOfVgroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); assert(index >= 0 && index < numOfVgroups); @@ -590,7 +590,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); // set the vgroup info - tscSetDnodeIpList(pSql, &pTableIdList->vgInfo); + tscSetDnodeEpSet(pSql, &pTableIdList->vgInfo); pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); int32_t numOfTables = taosArrayGetSize(pTableIdList->itemList); @@ -1323,7 +1323,7 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - tscSetDnodeIpList(pSql, &pTableMetaInfo->pTableMeta->vgroupInfo); + tscSetDnodeEpSet(pSql, &pTableMetaInfo->pTableMeta->vgroupInfo); return TSDB_CODE_SUCCESS; } @@ -1658,8 +1658,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); - if (pMetaMsg->sid < 0 || pMetaMsg->vgroup.numOfIps < 0) { - tscError("invalid meter vgId:%d, sid%d", pMetaMsg->vgroup.numOfIps, pMetaMsg->sid); + if (pMetaMsg->sid < 0 || pMetaMsg->vgroup.numOfEps < 0) { + tscError("invalid meter vgId:%d, sid%d", pMetaMsg->vgroup.numOfEps, pMetaMsg->sid); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -1673,8 +1673,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { return TSDB_CODE_TSC_INVALID_VALUE; } - for (int i = 0; i < pMetaMsg->vgroup.numOfIps; ++i) { - pMetaMsg->vgroup.ipAddr[i].port = htons(pMetaMsg->vgroup.ipAddr[i].port); + for (int i = 0; i < pMetaMsg->vgroup.numOfEps; ++i) { + pMetaMsg->vgroup.epAddr[i].port = htons(pMetaMsg->vgroup.epAddr[i].port); } SSchema* pSchema = pMetaMsg->schema; @@ -1850,10 +1850,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; pVgroups->vgId = htonl(pVgroups->vgId); - assert(pVgroups->numOfIps >= 1); + assert(pVgroups->numOfEps >= 1); - for (int32_t k = 0; k < pVgroups->numOfIps; ++k) { - pVgroups->ipAddr[k].port = htons(pVgroups->ipAddr[k].port); + for (int32_t k = 0; k < pVgroups->numOfEps; ++k) { + pVgroups->epAddr[k].port = htons(pVgroups->epAddr[k].port); } pMsg += size; @@ -1946,8 +1946,8 @@ int tscProcessConnectRsp(SSqlObj *pSql) { assert(len <= sizeof(pObj->db)); tstrncpy(pObj->db, temp, sizeof(pObj->db)); - if (pConnect->ipList.numOfIps > 0) - tscSetMgmtIpList(&pConnect->ipList); + if (pConnect->epSet.numOfEps > 0) + tscSetMgmtEpSet(&pConnect->epSet); strcpy(pObj->sversion, pConnect->serverVersion); pObj->writeAuth = pConnect->writeAuth; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index d5160cd3c6..6a14d3a65e 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -62,8 +62,8 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con } if (ip) { - if (tscSetMgmtIpListFromCfg(ip, NULL) < 0) return NULL; - if (port) tscMgmtIpSet.port[0] = port; + if (tscSetMgmtEpSetFromCfg(ip, NULL) < 0) return NULL; + if (port) tscMgmtEpSet.port[0] = port; } void *pDnodeConn = NULL; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index faa4a2488a..1dbc52efb0 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -458,7 +458,7 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr break; } } - assert(info.vgInfo.numOfIps != 0); + assert(info.vgInfo.numOfEps != 0); vgTables = taosArrayInit(4, sizeof(STableIdInfo)); info.itemList = vgTables; @@ -1600,8 +1600,8 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p // data in from current vnode is stored in cache and disk uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->num; - tscDebug("%p sub:%p all data retrieved from ip:%s, vgId:%d, numOfRows:%d, orderOfSub:%d", pParentSql, pSql, - pTableMetaInfo->vgroupList->vgroups[0].ipAddr[0].fqdn, pTableMetaInfo->vgroupList->vgroups[0].vgId, + tscDebug("%p sub:%p all data retrieved from ep:%s, vgId:%d, numOfRows:%d, orderOfSub:%d", pParentSql, pSql, + pTableMetaInfo->vgroupList->vgroups[0].epAddr[0].fqdn, pTableMetaInfo->vgroupList->vgroups[0].vgId, numOfRowsFromSubquery, idx); tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity); @@ -1719,8 +1719,8 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR assert(pRes->numOfRows == numOfRows); int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows); - tscDebug("%p sub:%p retrieve numOfRows:%" PRId64 " totalNumOfRows:%" PRIu64 " from ip:%s, orderOfSub:%d", pParentSql, pSql, - pRes->numOfRows, pState->numOfRetrievedRows, pSql->ipList.fqdn[pSql->ipList.inUse], idx); + tscDebug("%p sub:%p retrieve numOfRows:%" PRId64 " totalNumOfRows:%" PRIu64 " from ep:%s, orderOfSub:%d", pParentSql, pSql, + pRes->numOfRows, pState->numOfRetrievedRows, pSql->epSet.fqdn[pSql->epSet.inUse], idx); if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId32 " , current:%" PRId64, @@ -1828,8 +1828,8 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { return; } - tscTrace("%p sub:%p query complete, ip:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql, - pVgroup->ipAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex); + tscTrace("%p sub:%p query complete, ep:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql, + pVgroup->epAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex); if (pSql->res.qhandle == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode tscRetrieveFromDnodeCallBack(param, pSql, 0); diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 82cc8cc225..7722acf0d0 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -41,7 +41,7 @@ int tscNumOfThreads; static pthread_once_t tscinit = PTHREAD_ONCE_INIT; void taosInitNote(int numOfNoteLines, int maxNotes, char* lable); -void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet); +void tscUpdateEpSet(void *ahandle, SRpcEpSet *pEpSet); void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) { taosGetDisk(); @@ -116,8 +116,8 @@ void taos_init_imp() { taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note"); } - if (tscSetMgmtIpListFromCfg(tsFirst, tsSecond) < 0) { - tscError("failed to init mnode IP list"); + if (tscSetMgmtEpSetFromCfg(tsFirst, tsSecond) < 0) { + tscError("failed to init mnode EP list"); return; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 49d15f6499..4e6133663e 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2145,17 +2145,17 @@ char* strdup_throw(const char* str) { return p; } -int tscSetMgmtIpListFromCfg(const char *first, const char *second) { - tscMgmtIpSet.numOfIps = 0; - tscMgmtIpSet.inUse = 0; +int tscSetMgmtEpSetFromCfg(const char *first, const char *second) { + tscMgmtEpSet.numOfEps = 0; + tscMgmtEpSet.inUse = 0; if (first && first[0] != 0) { if (strlen(first) >= TSDB_EP_LEN) { terrno = TSDB_CODE_TSC_INVALID_FQDN; return -1; } - taosGetFqdnPortFromEp(first, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); - tscMgmtIpSet.numOfIps++; + taosGetFqdnPortFromEp(first, tscMgmtEpSet.fqdn[tscMgmtEpSet.numOfEps], &tscMgmtEpSet.port[tscMgmtEpSet.numOfEps]); + tscMgmtEpSet.numOfEps++; } if (second && second[0] != 0) { @@ -2163,11 +2163,11 @@ int tscSetMgmtIpListFromCfg(const char *first, const char *second) { terrno = TSDB_CODE_TSC_INVALID_FQDN; return -1; } - taosGetFqdnPortFromEp(second, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); - tscMgmtIpSet.numOfIps++; + taosGetFqdnPortFromEp(second, tscMgmtEpSet.fqdn[tscMgmtEpSet.numOfEps], &tscMgmtEpSet.port[tscMgmtEpSet.numOfEps]); + tscMgmtEpSet.numOfEps++; } - if ( tscMgmtIpSet.numOfIps == 0) { + if ( tscMgmtEpSet.numOfEps == 0) { terrno = TSDB_CODE_TSC_INVALID_FQDN; return -1; } diff --git a/src/dnode/inc/dnodeMgmt.h b/src/dnode/inc/dnodeMgmt.h index 092c06d84b..e8f4a0823f 100644 --- a/src/dnode/inc/dnodeMgmt.h +++ b/src/dnode/inc/dnodeMgmt.h @@ -35,8 +35,8 @@ void* dnodeGetVnodeTsdb(void *pVnode); void dnodeReleaseVnode(void *pVnode); void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell); -void dnodeGetMnodeIpSetForPeer(void *ipSet); -void dnodeGetMnodeIpSetForShell(void *ipSet); +void dnodeGetMnodeEpSetForPeer(void *epSet); +void dnodeGetMnodeEpSetForShell(void *epSet); #ifdef __cplusplus } diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 7d3ef9926d..9050b9c582 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -52,7 +52,7 @@ void * tsDnodeTmr = NULL; static void * tsStatusTimer = NULL; static uint32_t tsRebootTime; -static SRpcIpSet tsDMnodeIpSet = {0}; +static SRpcEpSet tsDMnodeEpSet = {0}; static SDMMnodeInfos tsDMnodeInfos = {0}; static SDMDnodeCfg tsDnodeCfg = {0}; static taos_qset tsMgmtQset = NULL; @@ -90,21 +90,21 @@ int32_t dnodeInitMgmt() { tsRebootTime = taosGetTimestampSec(); if (!dnodeReadMnodeInfos()) { - memset(&tsDMnodeIpSet, 0, sizeof(SRpcIpSet)); + memset(&tsDMnodeEpSet, 0, sizeof(SRpcEpSet)); memset(&tsDMnodeInfos, 0, sizeof(SDMMnodeInfos)); - tsDMnodeIpSet.numOfIps = 1; - taosGetFqdnPortFromEp(tsFirst, tsDMnodeIpSet.fqdn[0], &tsDMnodeIpSet.port[0]); + tsDMnodeEpSet.numOfEps = 1; + taosGetFqdnPortFromEp(tsFirst, tsDMnodeEpSet.fqdn[0], &tsDMnodeEpSet.port[0]); if (strcmp(tsSecond, tsFirst) != 0) { - tsDMnodeIpSet.numOfIps = 2; - taosGetFqdnPortFromEp(tsSecond, tsDMnodeIpSet.fqdn[1], &tsDMnodeIpSet.port[1]); + tsDMnodeEpSet.numOfEps = 2; + taosGetFqdnPortFromEp(tsSecond, tsDMnodeEpSet.fqdn[1], &tsDMnodeEpSet.port[1]); } } else { - tsDMnodeIpSet.inUse = tsDMnodeInfos.inUse; - tsDMnodeIpSet.numOfIps = tsDMnodeInfos.nodeNum; + tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse; + tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum; for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { - taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeIpSet.fqdn[i], &tsDMnodeIpSet.port[i]); + taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]); } } @@ -450,27 +450,27 @@ static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) { return taosCfgDynamicOptions(pCfg->config); } -void dnodeUpdateMnodeIpSetForPeer(SRpcIpSet *pIpSet) { - dInfo("mnode IP list for is changed, numOfIps:%d inUse:%d", pIpSet->numOfIps, pIpSet->inUse); - for (int i = 0; i < pIpSet->numOfIps; ++i) { - pIpSet->port[i] -= TSDB_PORT_DNODEDNODE; - dInfo("mnode index:%d %s:%u", i, pIpSet->fqdn[i], pIpSet->port[i]) +void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet) { + dInfo("mnode EP list for is changed, numOfEps:%d inUse:%d", pEpSet->numOfEps, pEpSet->inUse); + for (int i = 0; i < pEpSet->numOfEps; ++i) { + pEpSet->port[i] -= TSDB_PORT_DNODEDNODE; + dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i]) } - tsDMnodeIpSet = *pIpSet; + tsDMnodeEpSet = *pEpSet; } -void dnodeGetMnodeIpSetForPeer(void *ipSetRaw) { - SRpcIpSet *ipSet = ipSetRaw; - *ipSet = tsDMnodeIpSet; +void dnodeGetMnodeEpSetForPeer(void *epSetRaw) { + SRpcEpSet *epSet = epSetRaw; + *epSet = tsDMnodeEpSet; - for (int i=0; inumOfIps; ++i) - ipSet->port[i] += TSDB_PORT_DNODEDNODE; + for (int i=0; inumOfEps; ++i) + epSet->port[i] += TSDB_PORT_DNODEDNODE; } -void dnodeGetMnodeIpSetForShell(void *ipSetRaw) { - SRpcIpSet *ipSet = ipSetRaw; - *ipSet = tsDMnodeIpSet; +void dnodeGetMnodeEpSetForShell(void *epSetRaw) { + SRpcEpSet *epSet = epSetRaw; + *epSet = tsDMnodeEpSet; } static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { @@ -536,10 +536,10 @@ static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) { dInfo("mnode index:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp); } - tsDMnodeIpSet.inUse = tsDMnodeInfos.inUse; - tsDMnodeIpSet.numOfIps = tsDMnodeInfos.nodeNum; + tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse; + tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum; for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { - taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeIpSet.fqdn[i], &tsDMnodeIpSet.port[i]); + taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]); } dnodeSaveMnodeInfos(); @@ -549,10 +549,10 @@ static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) { static bool dnodeReadMnodeInfos() { char ipFile[TSDB_FILENAME_LEN*2] = {0}; - sprintf(ipFile, "%s/mnodeIpList.json", tsDnodeDir); + sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir); FILE *fp = fopen(ipFile, "r"); if (!fp) { - dDebug("failed to read mnodeIpList.json, file not exist"); + dDebug("failed to read mnodeEpSet.json, file not exist"); return false; } @@ -563,40 +563,40 @@ static bool dnodeReadMnodeInfos() { if (len <= 0) { free(content); fclose(fp); - dError("failed to read mnodeIpList.json, content is null"); + dError("failed to read mnodeEpSet.json, content is null"); return false; } content[len] = 0; cJSON* root = cJSON_Parse(content); if (root == NULL) { - dError("failed to read mnodeIpList.json, invalid json format"); + dError("failed to read mnodeEpSet.json, invalid json format"); goto PARSE_OVER; } cJSON* inUse = cJSON_GetObjectItem(root, "inUse"); if (!inUse || inUse->type != cJSON_Number) { - dError("failed to read mnodeIpList.json, inUse not found"); + dError("failed to read mnodeEpSet.json, inUse not found"); goto PARSE_OVER; } tsDMnodeInfos.inUse = inUse->valueint; cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum"); if (!nodeNum || nodeNum->type != cJSON_Number) { - dError("failed to read mnodeIpList.json, nodeNum not found"); + dError("failed to read mnodeEpSet.json, nodeNum not found"); goto PARSE_OVER; } tsDMnodeInfos.nodeNum = nodeNum->valueint; cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { - dError("failed to read mnodeIpList.json, nodeInfos not found"); + dError("failed to read mnodeEpSet.json, nodeInfos not found"); goto PARSE_OVER; } int size = cJSON_GetArraySize(nodeInfos); if (size != tsDMnodeInfos.nodeNum) { - dError("failed to read mnodeIpList.json, nodeInfos size not matched"); + dError("failed to read mnodeEpSet.json, nodeInfos size not matched"); goto PARSE_OVER; } @@ -606,14 +606,14 @@ static bool dnodeReadMnodeInfos() { cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); if (!nodeId || nodeId->type != cJSON_Number) { - dError("failed to read mnodeIpList.json, nodeId not found"); + dError("failed to read mnodeEpSet.json, nodeId not found"); goto PARSE_OVER; } tsDMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint; cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { - dError("failed to read mnodeIpList.json, nodeName not found"); + dError("failed to read mnodeEpSet.json, nodeName not found"); goto PARSE_OVER; } strncpy(tsDMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN); @@ -621,7 +621,7 @@ static bool dnodeReadMnodeInfos() { ret = true; - dInfo("read mnode iplist successed, numOfIps:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse); + dInfo("read mnode epSet successed, numOfEps:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse); for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { dInfo("mnode:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp); } @@ -635,7 +635,7 @@ PARSE_OVER: static void dnodeSaveMnodeInfos() { char ipFile[TSDB_FILENAME_LEN] = {0}; - sprintf(ipFile, "%s/mnodeIpList.json", tsDnodeDir); + sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir); FILE *fp = fopen(ipFile, "w"); if (!fp) return; @@ -663,11 +663,11 @@ static void dnodeSaveMnodeInfos() { fclose(fp); free(content); - dInfo("save mnode iplist successed"); + dInfo("save mnode epSet successed"); } char *dnodeGetMnodeMasterEp() { - return tsDMnodeInfos.nodeInfos[tsDMnodeIpSet.inUse].nodeEp; + return tsDMnodeInfos.nodeInfos[tsDMnodeEpSet.inUse].nodeEp; } void* dnodeGetMnodeInfos() { @@ -726,9 +726,9 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { .msgType = TSDB_MSG_TYPE_DM_STATUS }; - SRpcIpSet ipSet; - dnodeGetMnodeIpSetForPeer(&ipSet); - dnodeSendMsgToDnode(&ipSet, &rpcMsg); + SRpcEpSet epSet; + dnodeGetMnodeEpSetForPeer(&epSet); + dnodeSendMsgToDnode(&epSet, &rpcMsg); } static bool dnodeReadDnodeCfg() { @@ -817,20 +817,20 @@ void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) { SRpcConnInfo connInfo = {0}; rpcGetConnInfo(rpcMsg->handle, &connInfo); - SRpcIpSet ipSet = {0}; + SRpcEpSet epSet = {0}; if (forShell) { - dnodeGetMnodeIpSetForShell(&ipSet); + dnodeGetMnodeEpSetForShell(&epSet); } else { - dnodeGetMnodeIpSetForPeer(&ipSet); + dnodeGetMnodeEpSetForPeer(&epSet); } - dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfIps:%d inUse:%d", taosMsg[rpcMsg->msgType], - taosIpStr(connInfo.clientIp), connInfo.user, ipSet.numOfIps, ipSet.inUse); + dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType], + taosIpStr(connInfo.clientIp), connInfo.user, epSet.numOfEps, epSet.inUse); - for (int i = 0; i < ipSet.numOfIps; ++i) { - dDebug("mnode index:%d %s:%d", i, ipSet.fqdn[i], ipSet.port[i]); - ipSet.port[i] = htons(ipSet.port[i]); + for (int i = 0; i < epSet.numOfEps; ++i) { + dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]); + epSet.port[i] = htons(epSet.port[i]); } - rpcSendRedirectRsp(rpcMsg->handle, &ipSet); + rpcSendRedirectRsp(rpcMsg->handle, &epSet); } diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index 2a3436583f..b27f56a871 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -29,11 +29,11 @@ #include "dnodeVWrite.h" #include "dnodeMPeer.h" -extern void dnodeUpdateMnodeIpSetForPeer(SRpcIpSet *pIpSet); +extern void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet); static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); -static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcIpSet *); +static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *); static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg); -static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet); +static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet); static void *tsDnodeServerRpc = NULL; static void *tsDnodeClientRpc = NULL; @@ -83,7 +83,7 @@ void dnodeCleanupServer() { } } -static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { +static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { SRpcMsg rspMsg = { .handle = pMsg->handle, .pCont = NULL, @@ -148,9 +148,9 @@ void dnodeCleanupClient() { } } -static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { - if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pIpSet) { - dnodeUpdateMnodeIpSetForPeer(pIpSet); +static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { + if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pEpSet) { + dnodeUpdateMnodeEpSetForPeer(pEpSet); } if (dnodeProcessRspMsgFp[pMsg->msgType]) { @@ -166,12 +166,12 @@ void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) { dnodeProcessRspMsgFp[msgType] = fp; } -void dnodeSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg) { - rpcSendRequest(tsDnodeClientRpc, ipSet, rpcMsg); +void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) { + rpcSendRequest(tsDnodeClientRpc, epSet, rpcMsg); } void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { - SRpcIpSet ipSet = {0}; - dnodeGetMnodeIpSetForPeer(&ipSet); - rpcSendRecv(tsDnodeClientRpc, &ipSet, rpcMsg, rpcRsp); + SRpcEpSet epSet = {0}; + dnodeGetMnodeEpSetForPeer(&epSet); + rpcSendRecv(tsDnodeClientRpc, &epSet, rpcMsg, rpcRsp); } diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index d089495079..f9d137bb99 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -31,7 +31,7 @@ #include "dnodeShell.h" static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); -static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcIpSet *); +static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *); static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey); static void * tsDnodeShellRpc = NULL; static int32_t tsDnodeQueryReqNum = 0; @@ -108,7 +108,7 @@ void dnodeCleanupShell() { } } -void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { +void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { SRpcMsg rpcMsg = { .handle = pMsg->handle, .pCont = NULL, diff --git a/src/inc/dnode.h b/src/inc/dnode.h index b561c407a3..5a059c93a6 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -39,13 +39,13 @@ SDnodeStatisInfo dnodeGetStatisInfo(); bool dnodeIsFirstDeploy(); char * dnodeGetMnodeMasterEp(); -void dnodeGetMnodeIpSetForPeer(void *ipSet); -void dnodeGetMnodeIpSetForShell(void *ipSet); +void dnodeGetMnodeEpSetForPeer(void *epSet); +void dnodeGetMnodeEpSetForShell(void *epSet); void * dnodeGetMnodeInfos(); int32_t dnodeGetDnodeId(); void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)); -void dnodeSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg); +void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg); void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp); void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t sid); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index a367d6b93b..96386aab65 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -176,7 +176,7 @@ extern char *taosMsg[]; typedef struct { char fqdn[TSDB_FQDN_LEN]; uint16_t port; -} SIpAddr; +} SEpAddr; typedef struct { int32_t numOfVnodes; @@ -306,7 +306,7 @@ typedef struct { int8_t reserved1; int8_t reserved2; int32_t connId; - SRpcIpSet ipList; + SRpcEpSet epSet; } SCMConnectRsp; typedef struct { @@ -648,8 +648,8 @@ typedef struct SCMSTableVgroupMsg { typedef struct { int32_t vgId; - int8_t numOfIps; - SIpAddr ipAddr[TSDB_MAX_REPLICA]; + int8_t numOfEps; + SEpAddr epAddr[TSDB_MAX_REPLICA]; } SCMVgroupInfo; typedef struct { @@ -753,7 +753,7 @@ typedef struct { uint32_t onlineDnodes; uint32_t connId; int8_t killConnection; - SRpcIpSet ipList; + SRpcEpSet epSet; } SCMHeartBeatRsp; typedef struct { diff --git a/src/inc/trpc.h b/src/inc/trpc.h index d1adfb7494..1af6a5eb0f 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -28,12 +28,12 @@ extern "C" { extern int tsRpcHeadSize; -typedef struct SRpcIpSet { +typedef struct SRpcEpSet { int8_t inUse; - int8_t numOfIps; + int8_t numOfEps; uint16_t port[TSDB_MAX_REPLICA]; char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN]; -} SRpcIpSet; +} SRpcEpSet; typedef struct SRpcConnInfo { uint32_t clientIp; @@ -67,7 +67,7 @@ typedef struct SRpcInit { char *ckey; // ciphering key // call back to process incoming msg, code shall be ignored by server app - void (*cfp)(SRpcMsg *, SRpcIpSet *); + void (*cfp)(SRpcMsg *, SRpcEpSet *); // call back to retrieve the client auth info, for server app only int (*afp)(char *tableId, char *spi, char *encrypt, char *secret, char *ckey); @@ -78,11 +78,11 @@ void rpcClose(void *); void *rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); void *rpcReallocCont(void *ptr, int contLen); -void rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg); +void rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg); void rpcSendResponse(const SRpcMsg *pMsg); -void rpcSendRedirectRsp(void *pConn, const SRpcIpSet *pIpSet); +void rpcSendRedirectRsp(void *pConn, const SRpcEpSet *pEpSet); int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); -void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pReq, SRpcMsg *pRsp); +void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); int rpcReportProgress(void *pConn, char *pCont, int contLen); void rpcCancelRequest(void *pContext); diff --git a/src/kit/taosmigrate/taosmigrate.c b/src/kit/taosmigrate/taosmigrate.c index b80ca44a10..3ef73ef169 100644 --- a/src/kit/taosmigrate/taosmigrate.c +++ b/src/kit/taosmigrate/taosmigrate.c @@ -210,10 +210,10 @@ int32_t main(int32_t argc, char *argv[]) { (void)snprintf(mnodeWal, TSDB_FILENAME_LEN*2, "%s/mnode/wal/wal0", arguments.dataDir); walModWalFile(mnodeWal); - // 2. modfiy dnode config: mnodeIpList.json - char dnodeIpList[TSDB_FILENAME_LEN*2] = {0}; - (void)snprintf(dnodeIpList, TSDB_FILENAME_LEN*2, "%s/dnode/mnodeIpList.json", arguments.dataDir); - modDnodeIpList(dnodeIpList); + // 2. modfiy dnode config: mnodeEpSet.json + char dnodeEpSet[TSDB_FILENAME_LEN*2] = {0}; + (void)snprintf(dnodeEpSet, TSDB_FILENAME_LEN*2, "%s/dnode/mnodeEpSet.json", arguments.dataDir); + modDnodeEpSet(dnodeEpSet); // 3. modify vnode config: config.json char vnodeDir[TSDB_FILENAME_LEN*2] = {0}; diff --git a/src/kit/taosmigrate/taosmigrate.h b/src/kit/taosmigrate/taosmigrate.h index a0a02e651c..9fb3c92db2 100644 --- a/src/kit/taosmigrate/taosmigrate.h +++ b/src/kit/taosmigrate/taosmigrate.h @@ -71,7 +71,7 @@ int tSystemShell(const char * cmd); void taosMvFile(char* destFile, char *srcFile) ; void walModWalFile(char* walfile); SdnodeIfo* getDnodeInfo(int32_t dnodeId); -void modDnodeIpList(char* dnodeIpList); +void modDnodeEpSet(char* dnodeEpSet); void modAllVnode(char *vnodeDir); #endif diff --git a/src/kit/taosmigrate/taosmigrateDnodeCfg.c b/src/kit/taosmigrate/taosmigrateDnodeCfg.c index 263d5521e9..7f6fd03fea 100644 --- a/src/kit/taosmigrate/taosmigrateDnodeCfg.c +++ b/src/kit/taosmigrate/taosmigrateDnodeCfg.c @@ -23,10 +23,10 @@ static SDMMnodeInfos tsDnodeIpInfos = {0}; -static bool dnodeReadMnodeInfos(char* dnodeIpList) { - FILE *fp = fopen(dnodeIpList, "r"); +static bool dnodeReadMnodeInfos(char* dnodeEpSet) { + FILE *fp = fopen(dnodeEpSet, "r"); if (!fp) { - printf("failed to read mnodeIpList.json, file not exist\n"); + printf("failed to read mnodeEpSet.json, file not exist\n"); return false; } @@ -37,40 +37,40 @@ static bool dnodeReadMnodeInfos(char* dnodeIpList) { if (len <= 0) { free(content); fclose(fp); - printf("failed to read mnodeIpList.json, content is null\n"); + printf("failed to read mnodeEpSet.json, content is null\n"); return false; } content[len] = 0; cJSON* root = cJSON_Parse(content); if (root == NULL) { - printf("failed to read mnodeIpList.json, invalid json format\n"); + printf("failed to read mnodeEpSet.json, invalid json format\n"); goto PARSE_OVER; } cJSON* inUse = cJSON_GetObjectItem(root, "inUse"); if (!inUse || inUse->type != cJSON_Number) { - printf("failed to read mnodeIpList.json, inUse not found\n"); + printf("failed to read mnodeEpSet.json, inUse not found\n"); goto PARSE_OVER; } tsDnodeIpInfos.inUse = inUse->valueint; cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum"); if (!nodeNum || nodeNum->type != cJSON_Number) { - printf("failed to read mnodeIpList.json, nodeNum not found\n"); + printf("failed to read mnodeEpSet.json, nodeNum not found\n"); goto PARSE_OVER; } tsDnodeIpInfos.nodeNum = nodeNum->valueint; cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { - printf("failed to read mnodeIpList.json, nodeInfos not found\n"); + printf("failed to read mnodeEpSet.json, nodeInfos not found\n"); goto PARSE_OVER; } int size = cJSON_GetArraySize(nodeInfos); if (size != tsDnodeIpInfos.nodeNum) { - printf("failed to read mnodeIpList.json, nodeInfos size not matched\n"); + printf("failed to read mnodeEpSet.json, nodeInfos size not matched\n"); goto PARSE_OVER; } @@ -80,14 +80,14 @@ static bool dnodeReadMnodeInfos(char* dnodeIpList) { cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); if (!nodeId || nodeId->type != cJSON_Number) { - printf("failed to read mnodeIpList.json, nodeId not found\n"); + printf("failed to read mnodeEpSet.json, nodeId not found\n"); goto PARSE_OVER; } tsDnodeIpInfos.nodeInfos[i].nodeId = nodeId->valueint; cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { - printf("failed to read mnodeIpList.json, nodeName not found\n"); + printf("failed to read mnodeEpSet.json, nodeName not found\n"); goto PARSE_OVER; } strncpy(tsDnodeIpInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN); @@ -102,7 +102,7 @@ static bool dnodeReadMnodeInfos(char* dnodeIpList) { ret = true; - //printf("read mnode iplist successed, numOfIps:%d inUse:%d\n", tsDnodeIpInfos.nodeNum, tsDnodeIpInfos.inUse); + //printf("read mnode epSet successed, numOfEps:%d inUse:%d\n", tsDnodeIpInfos.nodeNum, tsDnodeIpInfos.inUse); //for (int32_t i = 0; i < tsDnodeIpInfos.nodeNum; i++) { // printf("mnode:%d, %s\n", tsDnodeIpInfos.nodeInfos[i].nodeId, tsDnodeIpInfos.nodeInfos[i].nodeEp); //} @@ -115,8 +115,8 @@ PARSE_OVER: } -static void dnodeSaveMnodeInfos(char* dnodeIpList) { - FILE *fp = fopen(dnodeIpList, "w"); +static void dnodeSaveMnodeInfos(char* dnodeEpSet) { + FILE *fp = fopen(dnodeEpSet, "w"); if (!fp) return; int32_t len = 0; @@ -143,13 +143,13 @@ static void dnodeSaveMnodeInfos(char* dnodeIpList) { fclose(fp); free(content); - printf("mod mnode iplist successed\n"); + printf("mod mnode epSet successed\n"); } -void modDnodeIpList(char* dnodeIpList) +void modDnodeEpSet(char* dnodeEpSet) { - (void)dnodeReadMnodeInfos(dnodeIpList); - dnodeSaveMnodeInfos(dnodeIpList); + (void)dnodeReadMnodeInfos(dnodeEpSet); + dnodeSaveMnodeInfos(dnodeEpSet); return; } diff --git a/src/mnode/inc/mnodeMnode.h b/src/mnode/inc/mnodeMnode.h index 1060907234..0976ea8acd 100644 --- a/src/mnode/inc/mnodeMnode.h +++ b/src/mnode/inc/mnodeMnode.h @@ -42,12 +42,12 @@ void mnodeIncMnodeRef(struct SMnodeObj *pMnode); void mnodeDecMnodeRef(struct SMnodeObj *pMnode); char * mnodeGetMnodeRoleStr(); -void mnodeGetMnodeIpSetForPeer(SRpcIpSet *ipSet); -void mnodeGetMnodeIpSetForShell(SRpcIpSet *ipSet); +void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet); +void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet); char* mnodeGetMnodeMasterEp(); void mnodeGetMnodeInfos(void *mnodes); -void mnodeUpdateMnodeIpSet(); +void mnodeUpdateMnodeEpSet(); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index 3f1da89605..9c5b201e93 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -44,12 +44,12 @@ int32_t mnodeGetAvailableVgroup(struct SMnodeMsg *pMsg, SVgObj **pVgroup, int32_ void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable); void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable); -void mnodeSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle); +void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle); void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle); void mnodeSendAlterVgroupMsg(SVgObj *pVgroup); -SRpcIpSet mnodeGetIpSetFromVgroup(SVgObj *pVgroup); -SRpcIpSet mnodeGetIpSetFromIp(char *ep); +SRpcEpSet mnodeGetEpSetFromVgroup(SVgObj *pVgroup); +SRpcEpSet mnodeGetEpSetFromIp(char *ep); #ifdef __cplusplus } diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 9085f1a9f7..7edba8662e 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -289,14 +289,14 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { } } - SRpcIpSet ipSet = mnodeGetIpSetFromIp(pCmCfgDnode->ep); + SRpcEpSet epSet = mnodeGetEpSetFromIp(pCmCfgDnode->ep); if (dnodeId != 0) { SDnodeObj *pDnode = mnodeGetDnode(dnodeId); if (pDnode == NULL) { mError("failed to cfg dnode, invalid dnodeId:%d", dnodeId); return TSDB_CODE_MND_DNODE_NOT_EXIST; } - ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp); + epSet = mnodeGetEpSetFromIp(pDnode->dnodeEp); mnodeDecDnodeRef(pDnode); } @@ -313,7 +313,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { }; mInfo("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user); - dnodeSendMsgToDnode(&ipSet, &rpcMdCfgDnodeMsg); + dnodeSendMsgToDnode(&epSet, &rpcMdCfgDnodeMsg); return TSDB_CODE_SUCCESS; } @@ -399,9 +399,9 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { SVgObj *pVgroup = mnodeGetVgroup(pVload->vgId); if (pVgroup == NULL) { - SRpcIpSet ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp); + SRpcEpSet epSet = mnodeGetEpSetFromIp(pDnode->dnodeEp); mInfo("dnode:%d, vgId:%d not exist in mnode, drop it", pDnode->dnodeId, pVload->vgId); - mnodeSendDropVnodeMsg(pVload->vgId, &ipSet, NULL); + mnodeSendDropVnodeMsg(pVload->vgId, &epSet, NULL); } else { mnodeUpdateVgroupStatus(pVgroup, pDnode, pVload); pAccess->vgId = htonl(pVload->vgId); diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index f74de2b325..d2f389ca0b 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -35,8 +35,8 @@ static void * tsMnodeSdb = NULL; static int32_t tsMnodeUpdateSize = 0; -static SRpcIpSet tsMnodeIpSetForShell; -static SRpcIpSet tsMnodeIpSetForPeer; +static SRpcEpSet tsMnodeEpSetForShell; +static SRpcEpSet tsMnodeEpSetForPeer; static SDMMnodeInfos tsMnodeInfos; static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); @@ -123,7 +123,7 @@ static int32_t mnodeMnodeActionRestored() { sdbFreeIter(pIter); } - mnodeUpdateMnodeIpSet(); + mnodeUpdateMnodeEpSet(); return TSDB_CODE_SUCCESS; } @@ -204,13 +204,13 @@ char *mnodeGetMnodeRoleStr(int32_t role) { } } -void mnodeUpdateMnodeIpSet() { - mInfo("update mnodes ipset, numOfIps:%d ", mnodeGetMnodesNum()); +void mnodeUpdateMnodeEpSet() { + mInfo("update mnodes epSet, numOfEps:%d ", mnodeGetMnodesNum()); mnodeMnodeWrLock(); - memset(&tsMnodeIpSetForShell, 0, sizeof(SRpcIpSet)); - memset(&tsMnodeIpSetForPeer, 0, sizeof(SRpcIpSet)); + memset(&tsMnodeEpSetForShell, 0, sizeof(SRpcEpSet)); + memset(&tsMnodeEpSetForPeer, 0, sizeof(SRpcEpSet)); memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos)); int32_t index = 0; @@ -222,20 +222,20 @@ void mnodeUpdateMnodeIpSet() { SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); if (pDnode != NULL) { - strcpy(tsMnodeIpSetForShell.fqdn[index], pDnode->dnodeFqdn); - tsMnodeIpSetForShell.port[index] = htons(pDnode->dnodePort); - mDebug("mnode:%d, for shell fqdn:%s %d", pDnode->dnodeId, tsMnodeIpSetForShell.fqdn[index], htons(tsMnodeIpSetForShell.port[index])); + strcpy(tsMnodeEpSetForShell.fqdn[index], pDnode->dnodeFqdn); + tsMnodeEpSetForShell.port[index] = htons(pDnode->dnodePort); + mDebug("mnode:%d, for shell fqdn:%s %d", pDnode->dnodeId, tsMnodeEpSetForShell.fqdn[index], htons(tsMnodeEpSetForShell.port[index])); - strcpy(tsMnodeIpSetForPeer.fqdn[index], pDnode->dnodeFqdn); - tsMnodeIpSetForPeer.port[index] = htons(pDnode->dnodePort + TSDB_PORT_DNODEDNODE); - mDebug("mnode:%d, for peer fqdn:%s %d", pDnode->dnodeId, tsMnodeIpSetForPeer.fqdn[index], htons(tsMnodeIpSetForPeer.port[index])); + strcpy(tsMnodeEpSetForPeer.fqdn[index], pDnode->dnodeFqdn); + tsMnodeEpSetForPeer.port[index] = htons(pDnode->dnodePort + TSDB_PORT_DNODEDNODE); + mDebug("mnode:%d, for peer fqdn:%s %d", pDnode->dnodeId, tsMnodeEpSetForPeer.fqdn[index], htons(tsMnodeEpSetForPeer.port[index])); tsMnodeInfos.nodeInfos[index].nodeId = htonl(pMnode->mnodeId); strcpy(tsMnodeInfos.nodeInfos[index].nodeEp, pDnode->dnodeEp); if (pMnode->role == TAOS_SYNC_ROLE_MASTER) { - tsMnodeIpSetForShell.inUse = index; - tsMnodeIpSetForPeer.inUse = index; + tsMnodeEpSetForShell.inUse = index; + tsMnodeEpSetForPeer.inUse = index; tsMnodeInfos.inUse = index; } @@ -248,23 +248,23 @@ void mnodeUpdateMnodeIpSet() { } tsMnodeInfos.nodeNum = index; - tsMnodeIpSetForShell.numOfIps = index; - tsMnodeIpSetForPeer.numOfIps = index; + tsMnodeEpSetForShell.numOfEps = index; + tsMnodeEpSetForPeer.numOfEps = index; sdbFreeIter(pIter); mnodeMnodeUnLock(); } -void mnodeGetMnodeIpSetForPeer(SRpcIpSet *ipSet) { +void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet) { mnodeMnodeRdLock(); - *ipSet = tsMnodeIpSetForPeer; + *epSet = tsMnodeEpSetForPeer; mnodeMnodeUnLock(); } -void mnodeGetMnodeIpSetForShell(SRpcIpSet *ipSet) { +void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet) { mnodeMnodeRdLock(); - *ipSet = tsMnodeIpSetForShell; + *epSet = tsMnodeEpSetForShell; mnodeMnodeUnLock(); } @@ -295,7 +295,7 @@ int32_t mnodeAddMnode(int32_t dnodeId) { code = TSDB_CODE_MND_SDB_ERROR; } - mnodeUpdateMnodeIpSet(); + mnodeUpdateMnodeEpSet(); return code; } @@ -308,7 +308,7 @@ void mnodeDropMnodeLocal(int32_t dnodeId) { mnodeDecMnodeRef(pMnode); } - mnodeUpdateMnodeIpSet(); + mnodeUpdateMnodeEpSet(); } int32_t mnodeDropMnode(int32_t dnodeId) { @@ -330,7 +330,7 @@ int32_t mnodeDropMnode(int32_t dnodeId) { sdbDecRef(tsMnodeSdb, pMnode); - mnodeUpdateMnodeIpSet(); + mnodeUpdateMnodeEpSet(); return code; } diff --git a/src/mnode/src/mnodePeer.c b/src/mnode/src/mnodePeer.c index 04f2889607..71b8b1ea84 100644 --- a/src/mnode/src/mnodePeer.c +++ b/src/mnode/src/mnodePeer.c @@ -53,14 +53,14 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) { if (!sdbIsMaster()) { SMnodeRsp *rpcRsp = &pMsg->rpcRsp; - SRpcIpSet *ipSet = rpcMallocCont(sizeof(SRpcIpSet)); - mnodeGetMnodeIpSetForPeer(ipSet); - rpcRsp->rsp = ipSet; - rpcRsp->len = sizeof(SRpcIpSet); + SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); + mnodeGetMnodeEpSetForPeer(epSet); + rpcRsp->rsp = epSet; + rpcRsp->len = sizeof(SRpcEpSet); - mDebug("%p, msg:%s in mpeer queue, will be redireced inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], ipSet->inUse); - for (int32_t i = 0; i < ipSet->numOfIps; ++i) { - mDebug("mnode index:%d ip:%s:%d", i, ipSet->fqdn[i], htons(ipSet->port[i])); + mDebug("%p, msg:%s in mpeer queue, will be redireced inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->inUse); + for (int32_t i = 0; i < epSet->numOfEps; ++i) { + mDebug("mnode index:%d ep:%s:%d", i, epSet->fqdn[i], htons(epSet->port[i])); } return TSDB_CODE_RPC_REDIRECT; diff --git a/src/mnode/src/mnodeRead.c b/src/mnode/src/mnodeRead.c index dfecf1f693..af2fed3408 100644 --- a/src/mnode/src/mnodeRead.c +++ b/src/mnode/src/mnodeRead.c @@ -49,14 +49,14 @@ int32_t mnodeProcessRead(SMnodeMsg *pMsg) { if (!sdbIsMaster()) { SMnodeRsp *rpcRsp = &pMsg->rpcRsp; - SRpcIpSet *ipSet = rpcMallocCont(sizeof(SRpcIpSet)); - mnodeGetMnodeIpSetForShell(ipSet); - rpcRsp->rsp = ipSet; - rpcRsp->len = sizeof(SRpcIpSet); + SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); + mnodeGetMnodeEpSetForShell(epSet); + rpcRsp->rsp = epSet; + rpcRsp->len = sizeof(SRpcEpSet); - mDebug("%p, msg:%s in mread queue, will be redireced, inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], ipSet->inUse); - for (int32_t i = 0; i < ipSet->numOfIps; ++i) { - mDebug("mnode index:%d ip:%s:%d", i, ipSet->fqdn[i], htons(ipSet->port[i])); + mDebug("%p, msg:%s in mread queue, will be redireced, inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->inUse); + for (int32_t i = 0; i < epSet->numOfEps; ++i) { + mDebug("mnode index:%d ep:%s:%d", i, epSet->fqdn[i], htons(epSet->port[i])); } return TSDB_CODE_RPC_REDIRECT; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index e08f8ca0db..0f72dbdec4 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -219,7 +219,7 @@ void sdbUpdateMnodeRoles() { } } - mnodeUpdateMnodeIpSet(); + mnodeUpdateMnodeEpSet(); } static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) { diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index a56ad34a25..e3d5b41be3 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -270,7 +270,7 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { pHBRsp->onlineDnodes = htonl(mnodeGetOnlinDnodesNum()); pHBRsp->totalDnodes = htonl(mnodeGetDnodesNum()); - mnodeGetMnodeIpSetForShell(&pHBRsp->ipList); + mnodeGetMnodeEpSetForShell(&pHBRsp->epSet); pMsg->rpcRsp.rsp = pHBRsp; pMsg->rpcRsp.len = sizeof(SCMHeartBeatRsp); @@ -335,7 +335,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { pConnectRsp->writeAuth = pUser->writeAuth; pConnectRsp->superAuth = pUser->superAuth; - mnodeGetMnodeIpSetForShell(&pConnectRsp->ipList); + mnodeGetMnodeEpSetForShell(&pConnectRsp->epSet); connect_over: if (code != TSDB_CODE_SUCCESS) { diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 80c4fc95ab..75ed442cd4 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -910,9 +910,9 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { mInfo("app:%p:%p, stable:%s, send drop stable msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, pVgroup->vgId); - SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pVgroup); + SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pVgroup); SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE}; - dnodeSendMsgToDnode(&ipSet, &rpcMsg); + dnodeSendMsgToDnode(&epSet, &rpcMsg); mnodeDecVgroupRef(pVgroup); } taosHashDestroyIter(pIter); @@ -1484,10 +1484,10 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { SDnodeObj *pDnode = pVgroup->vnodeGid[vn].pDnode; if (pDnode == NULL) break; - tstrncpy(pVgroupInfo->vgroups[vgSize].ipAddr[vn].fqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN); - pVgroupInfo->vgroups[vgSize].ipAddr[vn].port = htons(pDnode->dnodePort); + tstrncpy(pVgroupInfo->vgroups[vgSize].epAddr[vn].fqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN); + pVgroupInfo->vgroups[vgSize].epAddr[vn].port = htons(pDnode->dnodePort); - pVgroupInfo->vgroups[vgSize].numOfIps++; + pVgroupInfo->vgroups[vgSize].numOfEps++; } vgSize++; @@ -1615,7 +1615,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { return terrno; } - SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup); + SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pMsg->pVgroup); SRpcMsg rpcMsg = { .ahandle = pMsg, .pCont = pMDCreate, @@ -1624,7 +1624,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE }; - dnodeSendMsgToDnode(&ipSet, &rpcMsg); + dnodeSendMsgToDnode(&epSet, &rpcMsg); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -1788,7 +1788,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { pDrop->sid = htonl(pTable->sid); pDrop->uid = htobe64(pTable->uid); - SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup); + SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pMsg->pVgroup); mInfo("app:%p:%p, table:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId, pTable->vgId, pTable->sid, pTable->uid); @@ -1803,7 +1803,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { if (!needReturn) rpcMsg.ahandle = NULL; - dnodeSendMsgToDnode(&ipSet, &rpcMsg); + dnodeSendMsgToDnode(&epSet, &rpcMsg); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -1842,7 +1842,7 @@ static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) { } } - SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup); + SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pMsg->pVgroup); SRpcMsg rpcMsg = { .ahandle = pMsg, .pCont = pMDCreate, @@ -1854,7 +1854,7 @@ static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) { mDebug("app:%p:%p, ctable %s, send alter column msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, pMsg->pVgroup->vgId); - dnodeSendMsgToDnode(&ipSet, &rpcMsg); + dnodeSendMsgToDnode(&epSet, &rpcMsg); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -1996,9 +1996,9 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { for (int32_t i = 0; i < pMsg->pVgroup->numOfVnodes; ++i) { SDnodeObj *pDnode = mnodeGetDnode(pMsg->pVgroup->vnodeGid[i].dnodeId); if (pDnode == NULL) break; - strcpy(pMeta->vgroup.ipAddr[i].fqdn, pDnode->dnodeFqdn); - pMeta->vgroup.ipAddr[i].port = htons(pDnode->dnodePort + TSDB_PORT_DNODESHELL); - pMeta->vgroup.numOfIps++; + strcpy(pMeta->vgroup.epAddr[i].fqdn, pDnode->dnodeFqdn); + pMeta->vgroup.epAddr[i].port = htons(pDnode->dnodePort + TSDB_PORT_DNODESHELL); + pMeta->vgroup.numOfEps++; mnodeDecDnodeRef(pDnode); } pMeta->vgroup.vgId = htonl(pMsg->pVgroup->vgId); diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 5f24981b50..1de591df7c 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -317,9 +317,9 @@ void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVl } if (!dnodeExist) { - SRpcIpSet ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp); + SRpcEpSet epSet = mnodeGetEpSetFromIp(pDnode->dnodeEp); mError("vgId:%d, dnode:%d not exist in mnode, drop it", pVload->vgId, pDnode->dnodeId); - mnodeSendDropVnodeMsg(pVload->vgId, &ipSet, NULL); + mnodeSendDropVnodeMsg(pVload->vgId, &epSet, NULL); return; } @@ -809,29 +809,29 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { return pVnode; } -SRpcIpSet mnodeGetIpSetFromVgroup(SVgObj *pVgroup) { - SRpcIpSet ipSet = { - .numOfIps = pVgroup->numOfVnodes, +SRpcEpSet mnodeGetEpSetFromVgroup(SVgObj *pVgroup) { + SRpcEpSet epSet = { + .numOfEps = pVgroup->numOfVnodes, .inUse = 0, }; for (int i = 0; i < pVgroup->numOfVnodes; ++i) { - strcpy(ipSet.fqdn[i], pVgroup->vnodeGid[i].pDnode->dnodeFqdn); - ipSet.port[i] = pVgroup->vnodeGid[i].pDnode->dnodePort + TSDB_PORT_DNODEDNODE; + strcpy(epSet.fqdn[i], pVgroup->vnodeGid[i].pDnode->dnodeFqdn); + epSet.port[i] = pVgroup->vnodeGid[i].pDnode->dnodePort + TSDB_PORT_DNODEDNODE; } - return ipSet; + return epSet; } -SRpcIpSet mnodeGetIpSetFromIp(char *ep) { - SRpcIpSet ipSet; +SRpcEpSet mnodeGetEpSetFromIp(char *ep) { + SRpcEpSet epSet; - ipSet.numOfIps = 1; - ipSet.inUse = 0; - taosGetFqdnPortFromEp(ep, ipSet.fqdn[0], &ipSet.port[0]); - ipSet.port[0] += TSDB_PORT_DNODEDNODE; - return ipSet; + epSet.numOfEps = 1; + epSet.inUse = 0; + taosGetFqdnPortFromEp(ep, epSet.fqdn[0], &epSet.port[0]); + epSet.port[0] += TSDB_PORT_DNODEDNODE; + return epSet; } -static void mnodeSendAlterVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet) { +static void mnodeSendAlterVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet) { SMDAlterVnodeMsg *pAlter = mnodeBuildVnodeMsg(pVgroup); SRpcMsg rpcMsg = { .ahandle = NULL, @@ -840,21 +840,21 @@ static void mnodeSendAlterVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet) { .code = 0, .msgType = TSDB_MSG_TYPE_MD_ALTER_VNODE }; - dnodeSendMsgToDnode(ipSet, &rpcMsg); + dnodeSendMsgToDnode(epSet, &rpcMsg); } void mnodeSendAlterVgroupMsg(SVgObj *pVgroup) { mDebug("vgId:%d, send alter all vnodes msg, numOfVnodes:%d db:%s", pVgroup->vgId, pVgroup->numOfVnodes, pVgroup->dbName); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SRpcIpSet ipSet = mnodeGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); + SRpcEpSet epSet = mnodeGetEpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); mDebug("vgId:%d, index:%d, send alter vnode msg to dnode %s", pVgroup->vgId, i, pVgroup->vnodeGid[i].pDnode->dnodeEp); - mnodeSendAlterVnodeMsg(pVgroup, &ipSet); + mnodeSendAlterVnodeMsg(pVgroup, &epSet); } } -static void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) { +static void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet, void *ahandle) { SMDCreateVnodeMsg *pCreate = mnodeBuildVnodeMsg(pVgroup); SRpcMsg rpcMsg = { .ahandle = ahandle, @@ -863,17 +863,17 @@ static void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *aha .code = 0, .msgType = TSDB_MSG_TYPE_MD_CREATE_VNODE }; - dnodeSendMsgToDnode(ipSet, &rpcMsg); + dnodeSendMsgToDnode(epSet, &rpcMsg); } void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) { mDebug("vgId:%d, send create all vnodes msg, numOfVnodes:%d db:%s", pVgroup->vgId, pVgroup->numOfVnodes, pVgroup->dbName); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SRpcIpSet ipSet = mnodeGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); + SRpcEpSet epSet = mnodeGetEpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); mDebug("vgId:%d, index:%d, send create vnode msg to dnode %s, ahandle:%p", pVgroup->vgId, i, pVgroup->vnodeGid[i].pDnode->dnodeEp, ahandle); - mnodeSendCreateVnodeMsg(pVgroup, &ipSet, ahandle); + mnodeSendCreateVnodeMsg(pVgroup, &epSet, ahandle); } } @@ -926,7 +926,7 @@ static SMDDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) { return pDrop; } -void mnodeSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) { +void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle) { SMDDropVnodeMsg *pDrop = mnodeBuildDropVnodeMsg(vgId); SRpcMsg rpcMsg = { .ahandle = ahandle, @@ -935,16 +935,16 @@ void mnodeSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) { .code = 0, .msgType = TSDB_MSG_TYPE_MD_DROP_VNODE }; - dnodeSendMsgToDnode(ipSet, &rpcMsg); + dnodeSendMsgToDnode(epSet, &rpcMsg); } static void mnodeSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) { pVgroup->status = TAOS_VG_STATUS_DROPPING; // deleting mDebug("vgId:%d, send drop all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SRpcIpSet ipSet = mnodeGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); + SRpcEpSet epSet = mnodeGetEpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); mDebug("vgId:%d, send drop vnode msg to dnode:%d, ahandle:%p", pVgroup->vgId, pVgroup->vnodeGid[i].dnodeId, ahandle); - mnodeSendDropVnodeMsg(pVgroup->vgId, &ipSet, ahandle); + mnodeSendDropVnodeMsg(pVgroup->vgId, &epSet, ahandle); } } @@ -998,8 +998,8 @@ static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) { } mDebug("vgId:%d, send create vnode msg to dnode %s for vnode cfg msg", pVgroup->vgId, pDnode->dnodeEp); - SRpcIpSet ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp); - mnodeSendCreateVnodeMsg(pVgroup, &ipSet, NULL); + SRpcEpSet epSet = mnodeGetEpSetFromIp(pDnode->dnodeEp); + mnodeSendCreateVnodeMsg(pVgroup, &epSet, NULL); mnodeDecDnodeRef(pDnode); mnodeDecVgroupRef(pVgroup); diff --git a/src/mnode/src/mnodeWrite.c b/src/mnode/src/mnodeWrite.c index 326ed50981..ab3cfa2dad 100644 --- a/src/mnode/src/mnodeWrite.c +++ b/src/mnode/src/mnodeWrite.c @@ -49,16 +49,16 @@ int32_t mnodeProcessWrite(SMnodeMsg *pMsg) { if (!sdbIsMaster()) { SMnodeRsp *rpcRsp = &pMsg->rpcRsp; - SRpcIpSet *ipSet = rpcMallocCont(sizeof(SRpcIpSet)); - mnodeGetMnodeIpSetForShell(ipSet); - rpcRsp->rsp = ipSet; - rpcRsp->len = sizeof(SRpcIpSet); + SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); + mnodeGetMnodeEpSetForShell(epSet); + rpcRsp->rsp = epSet; + rpcRsp->len = sizeof(SRpcEpSet); mDebug("app:%p:%p, msg:%s will be redireced inUse:%d", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType], - ipSet->inUse); - for (int32_t i = 0; i < ipSet->numOfIps; ++i) { - mDebug("app:%p:%p, mnode index:%d ip:%s:%d", pMsg->rpcMsg.ahandle, pMsg, i, ipSet->fqdn[i], - htons(ipSet->port[i])); + epSet->inUse); + for (int32_t i = 0; i < epSet->numOfEps; ++i) { + mDebug("app:%p:%p, mnode index:%d ep:%s:%d", pMsg->rpcMsg.ahandle, pMsg, i, epSet->fqdn[i], + htons(epSet->port[i])); } return TSDB_CODE_RPC_REDIRECT; diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 5d67d5e615..2325d12d92 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -55,7 +55,7 @@ typedef struct { char secret[TSDB_KEY_LEN]; // secret for the link char ckey[TSDB_KEY_LEN]; // ciphering key - void (*cfp)(SRpcMsg *, SRpcIpSet *); + void (*cfp)(SRpcMsg *, SRpcEpSet *); int (*afp)(char *user, char *spi, char *encrypt, char *secret, char *ckey); int32_t refCount; @@ -71,7 +71,7 @@ typedef struct { typedef struct { SRpcInfo *pRpc; // associated SRpcInfo - SRpcIpSet ipSet; // ip list provided by app + SRpcEpSet epSet; // ip list provided by app void *ahandle; // handle provided by app void *signature; // for validation struct SRpcConn *pConn; // pConn allocated @@ -80,12 +80,12 @@ typedef struct { int32_t contLen; // content length int32_t code; // error code int16_t numOfTry; // number of try for different servers - int8_t oldInUse; // server IP inUse passed by app + int8_t oldInUse; // server EP inUse passed by app int8_t redirect; // flag to indicate redirect int8_t connType; // connection type SRpcMsg *pRsp; // for synchronous API tsem_t *pSem; // for synchronous API - SRpcIpSet *pSet; // for synchronous API + SRpcEpSet *pSet; // for synchronous API char msg[0]; // RpcHead starts from here } SRpcReqContext; @@ -355,7 +355,7 @@ void *rpcReallocCont(void *ptr, int contLen) { return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); } -void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg) { +void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) { SRpcInfo *pRpc = (SRpcInfo *)shandle; SRpcReqContext *pContext; @@ -364,11 +364,11 @@ void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg) { pContext->ahandle = pMsg->ahandle; pContext->signature = pContext; pContext->pRpc = (SRpcInfo *)shandle; - pContext->ipSet = *pIpSet; + pContext->epSet = *pEpSet; pContext->contLen = contLen; pContext->pCont = pMsg->pCont; pContext->msgType = pMsg->msgType; - pContext->oldInUse = pIpSet->inUse; + pContext->oldInUse = pEpSet->inUse; pContext->connType = RPC_CONN_UDPC; if (contLen > tsRpcMaxUdpSize) pContext->connType = RPC_CONN_TCPC; @@ -458,15 +458,15 @@ void rpcSendResponse(const SRpcMsg *pRsp) { return; } -void rpcSendRedirectRsp(void *thandle, const SRpcIpSet *pIpSet) { +void rpcSendRedirectRsp(void *thandle, const SRpcEpSet *pEpSet) { SRpcMsg rpcMsg; memset(&rpcMsg, 0, sizeof(rpcMsg)); - rpcMsg.contLen = sizeof(SRpcIpSet); + rpcMsg.contLen = sizeof(SRpcEpSet); rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); if (rpcMsg.pCont == NULL) return; - memcpy(rpcMsg.pCont, pIpSet, sizeof(SRpcIpSet)); + memcpy(rpcMsg.pCont, pEpSet, sizeof(SRpcEpSet)); rpcMsg.code = TSDB_CODE_RPC_REDIRECT; rpcMsg.handle = thandle; @@ -488,7 +488,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { return 0; } -void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { SRpcReqContext *pContext; pContext = (SRpcReqContext *) (pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); @@ -498,9 +498,9 @@ void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) tsem_init(&sem, 0, 0); pContext->pSem = &sem; pContext->pRsp = pRsp; - pContext->pSet = pIpSet; + pContext->pSet = pEpSet; - rpcSendRequest(shandle, pIpSet, pMsg); + rpcSendRequest(shandle, pEpSet, pMsg); tsem_wait(&sem); tsem_destroy(&sem); @@ -755,11 +755,11 @@ static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) { static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) { SRpcConn *pConn; SRpcInfo *pRpc = pContext->pRpc; - SRpcIpSet *pIpSet = &pContext->ipSet; + SRpcEpSet *pEpSet = &pContext->epSet; - pConn = rpcGetConnFromCache(pRpc->pCache, pIpSet->fqdn[pIpSet->inUse], pIpSet->port[pIpSet->inUse], pContext->connType); + pConn = rpcGetConnFromCache(pRpc->pCache, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); if ( pConn == NULL || pConn->user[0] == 0) { - pConn = rpcOpenConn(pRpc, pIpSet->fqdn[pIpSet->inUse], pIpSet->port[pIpSet->inUse], pContext->connType); + pConn = rpcOpenConn(pRpc, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); } if (pConn) { @@ -1020,16 +1020,16 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { pContext->pConn = NULL; if (pContext->pRsp) { // for synchronous API - memcpy(pContext->pSet, &pContext->ipSet, sizeof(SRpcIpSet)); + memcpy(pContext->pSet, &pContext->epSet, sizeof(SRpcEpSet)); memcpy(pContext->pRsp, pMsg, sizeof(SRpcMsg)); tsem_post(pContext->pSem); } else { // for asynchronous API - SRpcIpSet *pIpSet = NULL; - if (pContext->ipSet.inUse != pContext->oldInUse || pContext->redirect) - pIpSet = &pContext->ipSet; + SRpcEpSet *pEpSet = NULL; + if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect) + pEpSet = &pContext->epSet; - (*pRpc->cfp)(pMsg, pIpSet); + (*pRpc->cfp)(pMsg, pEpSet); } // free the request message @@ -1070,9 +1070,9 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { pConn->pContext = NULL; pConn->pReqMsg = NULL; - // for UDP, port may be changed by server, the port in ipSet shall be used for cache + // for UDP, port may be changed by server, the port in epSet shall be used for cache if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) { - rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->ipSet.port[pContext->ipSet.inUse], pConn->connType); + rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.port[pContext->epSet.inUse], pConn->connType); } else { rpcCloseConn(pConn); } @@ -1087,10 +1087,10 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { if (pHead->code == TSDB_CODE_RPC_REDIRECT) { pContext->numOfTry = 0; - memcpy(&pContext->ipSet, pHead->content, sizeof(pContext->ipSet)); - tDebug("%s, redirect is received, numOfIps:%d", pConn->info, pContext->ipSet.numOfIps); - for (int i=0; iipSet.numOfIps; ++i) - pContext->ipSet.port[i] = htons(pContext->ipSet.port[i]); + memcpy(&pContext->epSet, pHead->content, sizeof(pContext->epSet)); + tDebug("%s, redirect is received, numOfEps:%d", pConn->info, pContext->epSet.numOfEps); + for (int i=0; iepSet.numOfEps; ++i) + pContext->epSet.port[i] = htons(pContext->epSet.port[i]); rpcSendReqToServer(pRpc, pContext); rpcFreeCont(rpcMsg.pCont); } else if (pHead->code == TSDB_CODE_RPC_NOT_READY) { @@ -1269,7 +1269,7 @@ static void rpcProcessConnError(void *param, void *id) { tDebug("%s %p, connection error happens", pRpc->label, pContext->ahandle); - if (pContext->numOfTry >= pContext->ipSet.numOfIps) { + if (pContext->numOfTry >= pContext->epSet.numOfEps) { rpcMsg.msgType = pContext->msgType+1; rpcMsg.ahandle = pContext->ahandle; rpcMsg.code = pContext->code; @@ -1279,8 +1279,8 @@ static void rpcProcessConnError(void *param, void *id) { rpcNotifyClient(pContext, &rpcMsg); } else { // move to next IP - pContext->ipSet.inUse++; - pContext->ipSet.inUse = pContext->ipSet.inUse % pContext->ipSet.numOfIps; + pContext->epSet.inUse++; + pContext->epSet.inUse = pContext->epSet.inUse % pContext->epSet.numOfEps; rpcSendReqToServer(pRpc, pContext); } } diff --git a/src/rpc/test/rclient.c b/src/rpc/test/rclient.c index f8dbbedb11..e2d9d388f9 100644 --- a/src/rpc/test/rclient.c +++ b/src/rpc/test/rclient.c @@ -22,7 +22,7 @@ typedef struct { int index; - SRpcIpSet ipSet; + SRpcEpSet epSet; int num; int numOfReqs; int msgSize; @@ -32,11 +32,11 @@ typedef struct { void *pRpc; } SInfo; -static void processResponse(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { +static void processResponse(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { SInfo *pInfo = (SInfo *)pMsg->ahandle; tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, pMsg->code); - if (pIpSet) pInfo->ipSet = *pIpSet; + if (pEpSet) pInfo->epSet = *pEpSet; rpcFreeCont(pMsg->pCont); sem_post(&pInfo->rspSem); @@ -57,7 +57,7 @@ static void *sendRequest(void *param) { rpcMsg.ahandle = pInfo; rpcMsg.msgType = 1; tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); - rpcSendRequest(pInfo->pRpc, &pInfo->ipSet, &rpcMsg); + rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg); if ( pInfo->num % 20000 == 0 ) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); sem_wait(&pInfo->rspSem); @@ -71,7 +71,7 @@ static void *sendRequest(void *param) { int main(int argc, char *argv[]) { SRpcInit rpcInit; - SRpcIpSet ipSet; + SRpcEpSet epSet; int msgSize = 128; int numOfReqs = 0; int appThreads = 1; @@ -82,12 +82,12 @@ int main(int argc, char *argv[]) { pthread_attr_t thattr; // server info - ipSet.numOfIps = 1; - ipSet.inUse = 0; - ipSet.port[0] = 7000; - ipSet.port[1] = 7000; - strcpy(ipSet.fqdn[0], serverIp); - strcpy(ipSet.fqdn[1], "192.168.0.1"); + epSet.numOfEps = 1; + epSet.inUse = 0; + epSet.port[0] = 7000; + epSet.port[1] = 7000; + strcpy(epSet.fqdn[0], serverIp); + strcpy(epSet.fqdn[1], "192.168.0.1"); // client info memset(&rpcInit, 0, sizeof(rpcInit)); @@ -105,9 +105,9 @@ int main(int argc, char *argv[]) { for (int i=1; iindex = i; - pInfo->ipSet = ipSet; + pInfo->epSet = epSet; pInfo->numOfReqs = numOfReqs; pInfo->msgSize = msgSize; sem_init(&pInfo->rspSem, 0, 0); diff --git a/src/rpc/test/rsclient.c b/src/rpc/test/rsclient.c index 266697d555..6e6961784b 100644 --- a/src/rpc/test/rsclient.c +++ b/src/rpc/test/rsclient.c @@ -23,7 +23,7 @@ typedef struct { int index; - SRpcIpSet ipSet; + SRpcEpSet epSet; int num; int numOfReqs; int msgSize; @@ -51,7 +51,7 @@ static void *sendRequest(void *param) { rpcMsg.msgType = 1; tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); - rpcSendRecv(pInfo->pRpc, &pInfo->ipSet, &rpcMsg, &rspMsg); + rpcSendRecv(pInfo->pRpc, &pInfo->epSet, &rpcMsg, &rspMsg); // handle response if (rspMsg.code != 0) terror++; @@ -72,7 +72,7 @@ static void *sendRequest(void *param) { int main(int argc, char *argv[]) { SRpcInit rpcInit; - SRpcIpSet ipSet; + SRpcEpSet epSet; int msgSize = 128; int numOfReqs = 0; int appThreads = 1; @@ -83,12 +83,12 @@ int main(int argc, char *argv[]) { pthread_attr_t thattr; // server info - ipSet.numOfIps = 1; - ipSet.inUse = 0; - ipSet.port[0] = 7000; - ipSet.port[1] = 7000; - strcpy(ipSet.fqdn[0], serverIp); - strcpy(ipSet.fqdn[1], "192.168.0.1"); + epSet.numOfEps = 1; + epSet.inUse = 0; + epSet.port[0] = 7000; + epSet.port[1] = 7000; + strcpy(epSet.fqdn[0], serverIp); + strcpy(epSet.fqdn[1], "192.168.0.1"); // client info memset(&rpcInit, 0, sizeof(rpcInit)); @@ -106,9 +106,9 @@ int main(int argc, char *argv[]) { for (int i=1; iindex = i; - pInfo->ipSet = ipSet; + pInfo->epSet = epSet; pInfo->numOfReqs = numOfReqs; pInfo->msgSize = msgSize; sem_init(&pInfo->rspSem, 0, 0); diff --git a/src/rpc/test/rserver.c b/src/rpc/test/rserver.c index d06e9df64b..44c5cd6ab4 100644 --- a/src/rpc/test/rserver.c +++ b/src/rpc/test/rserver.c @@ -103,7 +103,7 @@ int retrieveAuthInfo(char *meterId, char *spi, char *encrypt, char *secret, char return ret; } -void processRequestMsg(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { +void processRequestMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { SRpcMsg *pTemp; pTemp = taosAllocateQitem(sizeof(SRpcMsg)); From 710ea32000c4221daf5896d5892c35b9b4baa866 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 19 Jul 2020 03:11:44 +0000 Subject: [PATCH 07/10] Bump lodash from 4.17.13 to 4.17.19 in /src/connector/grafana/tdengine Bumps [lodash](https://github.com/lodash/lodash) from 4.17.13 to 4.17.19. - [Release notes](https://github.com/lodash/lodash/releases) - [Commits](https://github.com/lodash/lodash/compare/4.17.13...4.17.19) Signed-off-by: dependabot[bot] --- src/connector/grafana/tdengine/package-lock.json | 8 ++++---- src/connector/grafana/tdengine/package.json | 2 +- src/connector/grafana/tdengine/yarn.lock | 12 ++++-------- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/connector/grafana/tdengine/package-lock.json b/src/connector/grafana/tdengine/package-lock.json index 7c8853b99c..013a005d8d 100644 --- a/src/connector/grafana/tdengine/package-lock.json +++ b/src/connector/grafana/tdengine/package-lock.json @@ -1,6 +1,6 @@ { "name": "TDengine", - "version": "1.0.0", + "version": "2.0.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -2302,9 +2302,9 @@ } }, "lodash": { - "version": "4.17.13", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.13.tgz", - "integrity": "sha512-vm3/XWXfWtRua0FkUyEHBZy8kCPjErNBT9fJx8Zvs+U6zjqPbTUOpkaoum3O5uiA8sm+yNMHXfYkTUHFoMxFNA==" + "version": "4.17.19", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", + "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==" }, "log-symbols": { "version": "2.2.0", diff --git a/src/connector/grafana/tdengine/package.json b/src/connector/grafana/tdengine/package.json index 678278ec47..b7e6512354 100644 --- a/src/connector/grafana/tdengine/package.json +++ b/src/connector/grafana/tdengine/package.json @@ -38,7 +38,7 @@ "q": "^1.5.0" }, "dependencies": { - "lodash": "^4.17.13", + "lodash": "^4.17.19", "yarn": "^1.22.0" }, "homepage": "https://github.com/taosdata/TDengine/tree/develop/src/connector/grafana/tdengine" diff --git a/src/connector/grafana/tdengine/yarn.lock b/src/connector/grafana/tdengine/yarn.lock index f785e4e478..00bc01df7a 100644 --- a/src/connector/grafana/tdengine/yarn.lock +++ b/src/connector/grafana/tdengine/yarn.lock @@ -1888,14 +1888,10 @@ locate-path@^3.0.0: p-locate "^3.0.0" path-exists "^3.0.0" -lodash@^4.17.10, lodash@^4.17.13, lodash@^4.17.4, lodash@^4.2.0, lodash@~4.17.10, lodash@~4.17.5: - version "4.17.13" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.13.tgz#0bdc3a6adc873d2f4e0c4bac285df91b64fc7b93" - -lodash@^4.17.15: - version "4.17.15" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548" - integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A== +lodash@^4.17.10, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.4, lodash@^4.2.0, lodash@~4.17.10, lodash@~4.17.5: + version "4.17.19" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.19.tgz#e48ddedbe30b3321783c5b4301fbd353bc1e4a4b" + integrity sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ== log-symbols@2.2.0: version "2.2.0" From 55e3352d7be4c3739bc6355f41c143bef0c37d33 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 19 Jul 2020 23:29:17 +0800 Subject: [PATCH 08/10] [td-225] refactor codes. opt resbuf perf. rename sources file --- src/client/inc/tsclient.h | 2 +- src/client/src/tscFunctionImpl.c | 8 +- src/client/src/tscServer.c | 63 ++--- src/query/inc/{qExecutor.h => qexecutor.h} | 6 +- src/query/inc/{qresultBuf.h => qresultbuf.h} | 24 +- src/query/inc/{qUtil.h => qutil.h} | 4 +- src/query/inc/tsqlfunction.h | 10 +- src/query/src/{qExecutor.c => qexecutor.c} | 132 +++++++---- .../src/{qFilterFunc.c => qfilterfunc.c} | 4 +- src/query/src/qresultBuf.c | 184 --------------- src/query/src/qresultbuf.c | 218 ++++++++++++++++++ src/query/src/{qUtil.c => qutil.c} | 8 +- src/query/tests/resultBufferTest.cpp | 2 +- src/tsdb/src/tsdbRead.c | 2 +- src/util/src/tutil.c | 2 +- 15 files changed, 365 insertions(+), 304 deletions(-) rename src/query/inc/{qExecutor.h => qexecutor.h} (99%) rename src/query/inc/{qresultBuf.h => qresultbuf.h} (75%) rename src/query/inc/{qUtil.h => qutil.h} (95%) rename src/query/src/{qExecutor.c => qexecutor.c} (98%) rename src/query/src/{qFilterFunc.c => qfilterfunc.c} (99%) delete mode 100644 src/query/src/qresultBuf.c create mode 100644 src/query/src/qresultbuf.c rename src/query/src/{qUtil.c => qutil.c} (98%) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index ef8184990d..3f548c18ca 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -30,7 +30,7 @@ extern "C" { #include "tsqlfunction.h" #include "tutil.h" -#include "qExecutor.h" +#include "qexecutor.h" #include "qsqlparser.h" #include "qtsbuf.h" #include "tcmdtype.h" diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 262b7ab3f6..fb15e4efd5 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -74,7 +74,7 @@ for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \ void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} void noop2(SQLFunctionCtx *UNUSED_PARAM(pCtx), int32_t UNUSED_PARAM(index)) {} -void doFinalizer(SQLFunctionCtx *pCtx) { resetResultInfo(GET_RES_INFO(pCtx)); } +void doFinalizer(SQLFunctionCtx *pCtx) { RESET_RESULT_INFO(GET_RES_INFO(pCtx)); } typedef struct tValuePair { tVariant v; @@ -330,12 +330,6 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } -/** - * the numOfRes should be kept, since it may be used later - * and allow the ResultInfo to be re initialized - */ -void resetResultInfo(SResultInfo *pResInfo) { pResInfo->initialized = false; } - void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf) { assert(pResInfo->interResultBuf == NULL); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 398dea9d09..2bc41a834d 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -242,52 +242,31 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { if (pEpSet) tscMgmtEpSet = *pEpSet; } - if (rpcMsg->pCont == NULL) { - rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - } else { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - // if (rpcMsg->code != TSDB_CODE_RPC_NETWORK_UNAVAIL) { - // if (pCmd->command == TSDB_SQL_CONNECT) { - // rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - // rpcFreeCont(rpcMsg->pCont); - // return; - // } + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - // if (pCmd->command == TSDB_SQL_HB) { - // rpcMsg->code = TSDB_CODE_RPC_NOT_READY; - // rpcFreeCont(rpcMsg->pCont); - // return; - // } + int32_t cmd = pCmd->command; + if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_FETCH || cmd == TSDB_SQL_INSERT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && + (rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || + rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID || + rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || + rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE)) { + tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry); - // if (pCmd->command == TSDB_SQL_META || pCmd->command == TSDB_SQL_DESCRIBE_TABLE || - // pCmd->command == TSDB_SQL_STABLEVGROUP || pCmd->command == TSDB_SQL_SHOW || - // pCmd->command == TSDB_SQL_RETRIEVE) { - // // get table meta/vgroup query will not retry, do nothing - // } - // } + // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema + if (rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { + pSql->cmd.submitSchema = 1; + } - if ((pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || pCmd->command == TSDB_SQL_INSERT || - pCmd->command == TSDB_SQL_UPDATE_TAGS_VAL) && - (rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID || - rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE)) { - tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry); - // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema - if (rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { - pSql->cmd.submitSchema = 1; - } + pSql->res.code = rpcMsg->code; // keep the previous error code + if (pSql->retry > pSql->maxRetry) { + tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry); + } else { + rpcMsg->code = tscRenewTableMeta(pSql, pTableMetaInfo->name); - pSql->res.code = rpcMsg->code; // keep the previous error code - if (pSql->retry > pSql->maxRetry) { - tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry); - } else { - rpcMsg->code = tscRenewTableMeta(pSql, pTableMetaInfo->name); - - // if there is an error occurring, proceed to the following error handling procedure. - // todo add test cases - if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - rpcFreeCont(rpcMsg->pCont); - return; - } + // if there is an error occurring, proceed to the following error handling procedure. + if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + rpcFreeCont(rpcMsg->pCont); + return; } } } diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qexecutor.h similarity index 99% rename from src/query/inc/qExecutor.h rename to src/query/inc/qexecutor.h index fb8750323f..97a7565de8 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qexecutor.h @@ -19,15 +19,15 @@ #include "hash.h" #include "qfill.h" -#include "qresultBuf.h" +#include "qresultbuf.h" #include "qsqlparser.h" #include "qtsbuf.h" +#include "query.h" #include "taosdef.h" #include "tarray.h" #include "tlockfree.h" #include "tsdb.h" #include "tsqlfunction.h" -#include "query.h" struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); @@ -158,7 +158,7 @@ typedef struct SQueryRuntimeEnv { SResultInfo* resultInfo; // todo refactor to merge with SWindowResInfo SQuery* pQuery; SQLFunctionCtx* pCtx; - int16_t numOfRowsPerPage; + int32_t numOfRowsPerPage; int16_t offset[TSDB_MAX_COLUMNS]; uint16_t scanFlag; // denotes reversed scan of data or not SFillInfo* pFillInfo; diff --git a/src/query/inc/qresultBuf.h b/src/query/inc/qresultbuf.h similarity index 75% rename from src/query/inc/qresultBuf.h rename to src/query/inc/qresultbuf.h index a323d530a2..f901474cc1 100644 --- a/src/query/inc/qresultBuf.h +++ b/src/query/inc/qresultbuf.h @@ -33,14 +33,19 @@ typedef struct SDiskbasedResultBuf { int32_t fd; // data file fd int32_t allocateId; // allocated page id int32_t incStep; // minimum allocated pages - char* pBuf; // mmap buffer pointer + void* pBuf; // mmap buffer pointer char* path; // file path - + int32_t pageSize; // current used page size + int32_t inMemPages; // numOfPages that are allocated in memory SHashObj* idsTable; // id hash table SIDList list; // for each id, there is a page id list + + void* iBuf; // inmemory buf + void* handle; // for debug purpose } SDiskbasedResultBuf; -#define DEFAULT_INTERN_BUF_PAGE_SIZE (8192L*5) +#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) +#define DEFAULT_INMEM_BUF_PAGES 10 /** * create disk-based result buffer @@ -49,7 +54,8 @@ typedef struct SDiskbasedResultBuf { * @param rowSize * @return */ -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize, void* handle); +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize, int32_t pagesize, + int32_t inMemPages, void* handle); /** * @@ -81,8 +87,14 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId); * @param id * @return */ -#define GET_RES_BUF_PAGE_BY_ID(buf, id) ((tFilePage*)((buf)->pBuf + DEFAULT_INTERN_BUF_PAGE_SIZE*(id))) - +//#define getResBufPage(buf, id) ((tFilePage*)((buf)->pBuf + (buf)->pageSize * (id))) +static FORCE_INLINE tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { + if (id < pResultBuf->inMemPages) { + return pResultBuf->iBuf + id * pResultBuf->pageSize; + } else { + return pResultBuf->pBuf + (id - pResultBuf->inMemPages) * pResultBuf->pageSize; + } +} /** * get the total buffer size in the format of disk file * @param pResultBuf diff --git a/src/query/inc/qUtil.h b/src/query/inc/qutil.h similarity index 95% rename from src/query/inc/qUtil.h rename to src/query/inc/qutil.h index 78ae7be030..7119cb75fe 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qutil.h @@ -49,7 +49,7 @@ static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int3 assert(pResult != NULL && pRuntimeEnv != NULL); SQuery *pQuery = pRuntimeEnv->pQuery; - tFilePage *page = GET_RES_BUF_PAGE_BY_ID(pRuntimeEnv->pResultBuf, pResult->pos.pageId); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId); int32_t realRowId = pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery); return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage + @@ -59,6 +59,4 @@ static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int3 __filter_func_t *getRangeFilterFuncArray(int32_t type); __filter_func_t *getValueFilterFuncArray(int32_t type); -bool supportPrefilter(int32_t type); - #endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index c687f01cbc..5ce9121cf1 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -255,7 +255,15 @@ extern int32_t functionCompatList[]; // compatible check array list bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval); -void resetResultInfo(SResultInfo *pResInfo); +/** + * the numOfRes should be kept, since it may be used later + * and allow the ResultInfo to be re initialized + */ +#define RESET_RESULT_INFO(_r) \ + do { \ + (_r)->initialized = false; \ + } while (0) + void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf); static FORCE_INLINE void initResultInfo(SResultInfo *pResInfo) { diff --git a/src/query/src/qExecutor.c b/src/query/src/qexecutor.c similarity index 98% rename from src/query/src/qExecutor.c rename to src/query/src/qexecutor.c index 42fc90f478..9348f702d0 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qexecutor.c @@ -20,12 +20,12 @@ #include "exception.h" #include "hash.h" -#include "qExecutor.h" -#include "qUtil.h" -#include "qresultBuf.h" +#include "qast.h" +#include "qexecutor.h" +#include "qresultbuf.h" #include "query.h" #include "queryLog.h" -#include "qast.h" +#include "qutil.h" #include "tlosertree.h" #include "tscompression.h" #include "ttime.h" @@ -233,9 +233,7 @@ bool isGroupbyNormalCol(SSqlGroupbyExpr *pGroupbyExpr) { for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { SColIndex *pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, i); if (pColIndex->flag == TSDB_COL_NORMAL) { - /* - * make sure the normal column locates at the second position if tbname exists in group by clause - */ + //make sure the normal column locates at the second position if tbname exists in group by clause if (pGroupbyExpr->numOfGroupCols > 1) { assert(pColIndex->colIndex > 0); } @@ -294,6 +292,17 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) { return false; } +bool isProjQuery(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functId = pQuery->pSelectExpr[i].base.functionId; + if (functId != TSDB_FUNC_PRJ && functId != TSDB_FUNC_TAGPRJ) { + return false; + } + } + + return true; +} + bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].base.functionId == TSDB_FUNC_TS_COMP; } static bool limitResults(SQueryRuntimeEnv* pRuntimeEnv) { @@ -470,7 +479,7 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult pData = getNewDataBuf(pResultBuf, sid, &pageId); } else { pageId = getLastPageId(list); - pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, pageId); + pData = getResBufPage(pResultBuf, pageId); if (pData->num >= numOfRowsPerPage) { pData = getNewDataBuf(pResultBuf, sid, &pageId); @@ -1003,7 +1012,6 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat case TSDB_DATA_TYPE_BIGINT: v = GET_INT64_VAL(pData); break; } -// assert(pRuntimeEnv->windowResInfo.hashList->size <= 2); SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pData, bytes, true); if (pWindowRes == NULL) { return -1; @@ -1201,9 +1209,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS continue; } - // interval window query + // interval window query, decide the time window according to the primary timestamp if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - // decide the time window according to the primary timestamp int64_t ts = tsCols[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); @@ -1225,8 +1232,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS while (1) { GET_NEXT_TIMEWINDOW(pQuery, &nextWin); - if (/*pWindowResInfo->startTime > nextWin.skey ||*/ - (nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + if ((nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || (nextWin.skey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { break; } @@ -1484,6 +1490,8 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order goto _clean; } + qDebug("QInfo:%p setup runtime env1", GET_QINFO_ADDR(pRuntimeEnv)); + pRuntimeEnv->offset[0] = 0; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base; @@ -1528,6 +1536,8 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order } } + qDebug("QInfo:%p setup runtime env2", GET_QINFO_ADDR(pRuntimeEnv)); + // set the order information for top/bottom query int32_t functionId = pCtx->functionId; @@ -1548,17 +1558,25 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order } } + qDebug("QInfo:%p setup runtime env3", GET_QINFO_ADDR(pRuntimeEnv)); + char* buf = (char*) pRuntimeEnv->resultInfo + sizeof(SResultInfo) * pQuery->numOfOutput; // set the intermediate result output buffer setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery, buf); + qDebug("QInfo:%p setup runtime env4", GET_QINFO_ADDR(pRuntimeEnv)); + // if it is group by normal column, do not set output buffer, the output buffer is pResult - if (!isGroupbyNormalCol(pQuery->pGroupbyExpr) && !pRuntimeEnv->stableQuery) { + if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery) { resetCtxOutputBuf(pRuntimeEnv); } + qDebug("QInfo:%p setup runtime env5", GET_QINFO_ADDR(pRuntimeEnv)); + setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx); + + qDebug("QInfo:%p init completed", GET_QINFO_ADDR(pRuntimeEnv)); return TSDB_CODE_SUCCESS; _clean: @@ -1910,9 +1928,20 @@ static int32_t getInitialPageNum(SQInfo *pQInfo) { return num; } -static FORCE_INLINE int32_t getNumOfRowsInResultPage(SQuery *pQuery, bool topBotQuery, bool isSTableQuery) { - int32_t rowSize = pQuery->rowSize * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, topBotQuery, isSTableQuery); - return (DEFAULT_INTERN_BUF_PAGE_SIZE - sizeof(tFilePage)) / rowSize; +static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, int32_t* rowsize) { + SQuery* pQuery = pRuntimeEnv->pQuery; + + *rowsize = pQuery->rowSize * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery); + int32_t overhead = sizeof(tFilePage); + + // one page contains at least two rows + *ps = DEFAULT_INTERN_BUF_PAGE_SIZE; + while(((*rowsize) * 2) > (*ps) - overhead) { + *ps = (*ps << 1u); + } + + pRuntimeEnv->numOfRowsPerPage = ((*ps) - sizeof(tFilePage)) / (*rowsize); + } #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) @@ -2038,8 +2067,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, } else { // check if this data block is required to load // Calculate all time windows that are overlapping or contain current data block. - // If current data block is contained by all possible time window, loading current - // data block is not needed. + // If current data block is contained by all possible time window, do not load current data block. if (QUERY_IS_INTERVAL_QUERY(pQuery) && overlapWithTimeWindow(pQuery, pBlockInfo)) { status = BLK_DATA_ALL_NEEDED; } @@ -2424,7 +2452,7 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes pCtx[i].aOutputBuf = pCtx[i].aOutputBuf + pCtx[i].outputBytes; pCtx[i].currentStage = FIRST_STAGE_MERGE; - resetResultInfo(pCtx[i].resultInfo); + RESET_RESULT_INFO(pCtx[i].resultInfo); aAggs[functionId].init(&pCtx[i]); } @@ -2661,7 +2689,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { int32_t size = taosArrayGetSize(list); for (int32_t i = 0; i < size; ++i) { int32_t* pgId = taosArrayGet(list, i); - tFilePage *pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, *pgId); + tFilePage *pData = getResBufPage(pResultBuf, *pgId); total += pData->num; } @@ -2670,7 +2698,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { int32_t offset = 0; for (int32_t j = 0; j < size; ++j) { int32_t* pgId = taosArrayGet(list, j); - tFilePage *pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, *pgId); + tFilePage *pData = getResBufPage(pResultBuf, *pgId); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; @@ -2860,10 +2888,10 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { SQuery * pQuery = pRuntimeEnv->pQuery; SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - int32_t capacity = (DEFAULT_INTERN_BUF_PAGE_SIZE - sizeof(tFilePage)) / pQuery->rowSize; // the base value for group result, since the maximum number of table for each vnode will not exceed 100,000. int32_t pageId = -1; + int32_t capacity = pResultBuf->numOfRowsPerPage; int32_t remain = pQuery->sdata[0]->num; int32_t offset = 0; @@ -3033,7 +3061,7 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { * set the output buffer information and intermediate buffer * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. */ - resetResultInfo(&pRuntimeEnv->resultInfo[i]); + RESET_RESULT_INFO(&pRuntimeEnv->resultInfo[i]); pCtx->resultInfo = &pRuntimeEnv->resultInfo[i]; // set the timestamp output buffer for top/bottom/diff query @@ -3072,7 +3100,7 @@ void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { pRuntimeEnv->pCtx[j].ptsOutputBuf += TSDB_KEYSIZE * output; } - resetResultInfo(pRuntimeEnv->pCtx[j].resultInfo); + RESET_RESULT_INFO(pRuntimeEnv->pCtx[j].resultInfo); } } @@ -3326,8 +3354,8 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { if (pRuntimeEnv->pSecQueryHandle == NULL) { longjmp(pRuntimeEnv->env, terrno); } - pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; + pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); pRuntimeEnv->scanFlag = REPEAT_SCAN; @@ -3462,7 +3490,6 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { return; } - int32_t GROUPRESULTID = 1; SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, sizeof(groupIndex), true); if (pWindowRes == NULL) { @@ -3474,7 +3501,7 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { * all group belong to one result set, and each group result has different group id so set the id to be one */ if (pWindowRes->pos.pageId == -1) { - if (addNewWindowResultBuf(pWindowRes, pRuntimeEnv->pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage) != + if (addNewWindowResultBuf(pWindowRes, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage) != TSDB_CODE_SUCCESS) { return; } @@ -4170,7 +4197,6 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) { } int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bool isSTableQuery) { - qDebug("QInfo:%p start to init qhandle", pQInfo); SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; int32_t code = TSDB_CODE_SUCCESS; @@ -4209,33 +4235,42 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo return code; } - pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, pRuntimeEnv->topBotQuery, isSTableQuery); + int32_t ps = DEFAULT_PAGE_SIZE; + int32_t rowsize = 0; + getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); if (isSTableQuery && !onlyQueryTags(pRuntimeEnv->pQuery)) { - int32_t rows = getInitialPageNum(pQInfo); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize, pQInfo); + int32_t numOfPages = getInitialPageNum(pQInfo); + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, numOfPages, rowsize, ps, numOfPages, pQInfo); if (code != TSDB_CODE_SUCCESS) { return code; } if (!QUERY_IS_INTERVAL_QUERY(pQuery)) { int16_t type = TSDB_DATA_TYPE_NULL; + int32_t threshold = 0; if (pRuntimeEnv->groupbyNormalCol) { // group by columns not tags; type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); + threshold = 4000; } else { type = TSDB_DATA_TYPE_INT; // group id + threshold = GET_NUM_OF_TABLEGROUP(pQInfo); + if (threshold < 8) { + threshold = 8; + } } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 32, 4096, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 8, threshold, type); if (code != TSDB_CODE_SUCCESS) { return code; } } - } else if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { - int32_t rows = getInitialPageNum(pQInfo); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize, pQInfo); + int32_t numOfResultRows = getInitialPageNum(pQInfo); + getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); + + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, numOfResultRows, rowsize, ps, numOfResultRows, pQInfo); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4247,7 +4282,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo type = TSDB_DATA_TYPE_TIMESTAMP; } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, rows, 4096, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, numOfResultRows, 4096, type); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -5705,13 +5740,18 @@ static void calResultBufSize(SQuery* pQuery) { const int32_t RESULT_MSG_MIN_ROWS = 8192; const float RESULT_THRESHOLD_RATIO = 0.85; - int32_t numOfRes = RESULT_MSG_MIN_SIZE / pQuery->rowSize; - if (numOfRes < RESULT_MSG_MIN_ROWS) { - numOfRes = RESULT_MSG_MIN_ROWS; - } + if (isProjQuery(pQuery)) { + int32_t numOfRes = RESULT_MSG_MIN_SIZE / pQuery->rowSize; + if (numOfRes < RESULT_MSG_MIN_ROWS) { + numOfRes = RESULT_MSG_MIN_ROWS; + } - pQuery->rec.capacity = numOfRes; - pQuery->rec.threshold = numOfRes * RESULT_THRESHOLD_RATIO; + pQuery->rec.capacity = numOfRes; + pQuery->rec.threshold = numOfRes * RESULT_THRESHOLD_RATIO; + } else { // in case of non-prj query, a smaller output buffer will be used. + pQuery->rec.capacity = 4096; + pQuery->rec.threshold = pQuery->rec.capacity * RESULT_THRESHOLD_RATIO; + } } static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, @@ -5723,6 +5763,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, if (pQInfo == NULL) { goto _cleanup_qinfo; } + // to make sure third party won't overwrite this structure pQInfo->signature = pQInfo; pQInfo->tableGroupInfo = *pTableGroupInfo; @@ -6041,11 +6082,10 @@ static void freeQInfo(SQInfo *pQInfo) { tfree(pQuery->sdata); tfree(pQuery); + pQInfo->signature = 0; qDebug("QInfo:%p QInfo is freed", pQInfo); - // destroy signature, in order to avoid the query process pass the object safety check - memset(pQInfo, 0, sizeof(SQInfo)); tfree(pQInfo); } @@ -6209,7 +6249,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo } int64_t el = taosGetTimestampUs() - st; - qDebug("qmsg:%p tag filter completed, elapsed time:%"PRId64"us", pQueryMsg, el); + qDebug("qmsg:%p tag filter completed, numOfTables:%zu, elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); } else { assert(0); } diff --git a/src/query/src/qFilterFunc.c b/src/query/src/qfilterfunc.c similarity index 99% rename from src/query/src/qFilterFunc.c rename to src/query/src/qfilterfunc.c index 1a95b9fe21..6f336434e1 100644 --- a/src/query/src/qFilterFunc.c +++ b/src/query/src/qfilterfunc.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "qExecutor.h" +#include "qexecutor.h" #include "taosmsg.h" #include "tcompare.h" #include "tsqlfunction.h" @@ -554,5 +554,3 @@ __filter_func_t* getValueFilterFuncArray(int32_t type) { default: return NULL; } } - -bool supportPrefilter(int32_t type) { return type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR; } diff --git a/src/query/src/qresultBuf.c b/src/query/src/qresultBuf.c deleted file mode 100644 index ae1a95179b..0000000000 --- a/src/query/src/qresultBuf.c +++ /dev/null @@ -1,184 +0,0 @@ -#include "qresultBuf.h" -#include "hash.h" -#include "qextbuffer.h" -#include "taoserror.h" -#include "queryLog.h" - -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize, void* handle) { - *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf)); - SDiskbasedResultBuf* pResBuf = *pResultBuf; - if (pResBuf == NULL) { - return TSDB_CODE_COM_OUT_OF_MEMORY; - } - - pResBuf->numOfRowsPerPage = (DEFAULT_INTERN_BUF_PAGE_SIZE - sizeof(tFilePage)) / rowSize; - pResBuf->numOfPages = size; - - pResBuf->totalBufSize = pResBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE; - pResBuf->incStep = 4; - - // init id hash table - pResBuf->idsTable = taosHashInit(size, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); - pResBuf->list = taosArrayInit(size, POINTER_BYTES); - - char path[4096] = {0}; - getTmpfilePath("tsdb_qbuf", path); - pResBuf->path = strdup(path); - - pResBuf->fd = open(pResBuf->path, O_CREAT | O_RDWR, 0666); - if (!FD_VALID(pResBuf->fd)) { - qError("failed to create tmp file: %s on disk. %s", pResBuf->path, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - int32_t ret = ftruncate(pResBuf->fd, pResBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE); - if (ret != TSDB_CODE_SUCCESS) { - qError("failed to create tmp file: %s on disk. %s", pResBuf->path, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - pResBuf->pBuf = mmap(NULL, pResBuf->totalBufSize, PROT_READ | PROT_WRITE, MAP_SHARED, pResBuf->fd, 0); - if (pResBuf->pBuf == MAP_FAILED) { - qError("QInfo:%p failed to map temp file: %s. %s", handle, pResBuf->path, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - qDebug("QInfo:%p create tmp file for output result:%s, %" PRId64 "bytes", handle, pResBuf->path, - pResBuf->totalBufSize); - - return TSDB_CODE_SUCCESS; -} - -int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->idsTable); } - -int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; } - -static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t numOfPages) { - assert(pResultBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE == pResultBuf->totalBufSize); - - int32_t ret = munmap(pResultBuf->pBuf, pResultBuf->totalBufSize); - pResultBuf->numOfPages += numOfPages; - - /* - * disk-based output buffer is exhausted, try to extend the disk-based buffer, the available disk space may - * be insufficient - */ - ret = ftruncate(pResultBuf->fd, pResultBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE); - if (ret != 0) { - // dError("QInfo:%p failed to create intermediate result output file:%s. %s", pQInfo, pSupporter->extBufFile, - // strerror(errno)); - return TSDB_CODE_QRY_NO_DISKSPACE; - } - - pResultBuf->totalBufSize = pResultBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE; - pResultBuf->pBuf = mmap(NULL, pResultBuf->totalBufSize, PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0); - - if (pResultBuf->pBuf == MAP_FAILED) { - // dError("QInfo:%p failed to map temp file: %s. %s", pQInfo, pSupporter->extBufFile, strerror(errno)); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE bool noMoreAvailablePages(SDiskbasedResultBuf* pResultBuf) { - return (pResultBuf->allocateId == pResultBuf->numOfPages - 1); -} - -static FORCE_INLINE int32_t getGroupIndex(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - assert(pResultBuf != NULL); - - char* p = taosHashGet(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t)); - if (p == NULL) { // it is a new group id - return -1; - } - - int32_t slot = GET_INT32_VAL(p); - assert(slot >= 0 && slot < taosHashGetSize(pResultBuf->idsTable)); - - return slot; -} - -static int32_t addNewGroupId(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - int32_t num = getNumOfResultBufGroupId(pResultBuf); // the num is the newest allocated group id slot - taosHashPut(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t), &num, sizeof(int32_t)); - - SArray* pa = taosArrayInit(1, sizeof(int32_t)); - taosArrayPush(pResultBuf->list, &pa); - - assert(taosArrayGetSize(pResultBuf->list) == taosHashGetSize(pResultBuf->idsTable)); - return num; -} - -static void registerPageId(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { - int32_t slot = getGroupIndex(pResultBuf, groupId); - if (slot < 0) { - slot = addNewGroupId(pResultBuf, groupId); - } - - SIDList pList = taosArrayGetP(pResultBuf->list, slot); - taosArrayPush(pList, &pageId); -} - -tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) { - if (noMoreAvailablePages(pResultBuf)) { - if (extendDiskFileSize(pResultBuf, pResultBuf->incStep) != TSDB_CODE_SUCCESS) { - return NULL; - } - } - - // register new id in this group - *pageId = (pResultBuf->allocateId++); - registerPageId(pResultBuf, groupId, *pageId); - - tFilePage* page = GET_RES_BUF_PAGE_BY_ID(pResultBuf, *pageId); - - // clear memory for the new page - memset(page, 0, DEFAULT_INTERN_BUF_PAGE_SIZE); - - return page; -} - -int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; } - -SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - int32_t slot = getGroupIndex(pResultBuf, groupId); - if (slot < 0) { - return taosArrayInit(1, sizeof(int32_t)); - } else { - return taosArrayGetP(pResultBuf->list, slot); - } -} - -void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) { - if (pResultBuf == NULL) { - return; - } - - if (FD_VALID(pResultBuf->fd)) { - close(pResultBuf->fd); - } - - qDebug("QInfo:%p disk-based output buffer closed, %" PRId64 " bytes, file:%s", handle, pResultBuf->totalBufSize, pResultBuf->path); - munmap(pResultBuf->pBuf, pResultBuf->totalBufSize); - unlink(pResultBuf->path); - - tfree(pResultBuf->path); - - size_t size = taosArrayGetSize(pResultBuf->list); - for (int32_t i = 0; i < size; ++i) { - SArray* pa = taosArrayGetP(pResultBuf->list, i); - taosArrayDestroy(pa); - } - - taosArrayDestroy(pResultBuf->list); - taosHashCleanup(pResultBuf->idsTable); - - tfree(pResultBuf); -} - -int32_t getLastPageId(SIDList pList) { - size_t size = taosArrayGetSize(pList); - return *(int32_t*) taosArrayGet(pList, size - 1); -} - diff --git a/src/query/src/qresultbuf.c b/src/query/src/qresultbuf.c new file mode 100644 index 0000000000..646d8b7986 --- /dev/null +++ b/src/query/src/qresultbuf.c @@ -0,0 +1,218 @@ +#include "qresultbuf.h" +#include "hash.h" +#include "qextbuffer.h" +#include "queryLog.h" +#include "taoserror.h" + +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize, + int32_t pagesize, int32_t inMemPages, void* handle) { + + *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf)); + SDiskbasedResultBuf* pResBuf = *pResultBuf; + if (pResBuf == NULL) { + return TSDB_CODE_COM_OUT_OF_MEMORY; + } + + pResBuf->pageSize = pagesize; + pResBuf->numOfPages = inMemPages; // all pages are in buffer in the first place + pResBuf->inMemPages = inMemPages; + assert(inMemPages <= numOfPages); + + pResBuf->numOfRowsPerPage = (pagesize - sizeof(tFilePage)) / rowSize; + + pResBuf->totalBufSize = pResBuf->numOfPages * pagesize; + pResBuf->incStep = 4; + pResBuf->allocateId = -1; + + pResBuf->iBuf = calloc(pResBuf->inMemPages, pResBuf->pageSize); + + // init id hash table + pResBuf->idsTable = taosHashInit(numOfPages, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + pResBuf->list = taosArrayInit(numOfPages, POINTER_BYTES); + + char path[PATH_MAX] = {0}; + getTmpfilePath("tsdb_qbuf", path); + pResBuf->path = strdup(path); + + pResBuf->fd = FD_INITIALIZER; + pResBuf->pBuf = NULL; + + qDebug("QInfo:%p create resBuf for output, page size:%d, initial pages:%d, %" PRId64 "bytes", handle, + pResBuf->pageSize, pResBuf->numOfPages, pResBuf->totalBufSize); + + return TSDB_CODE_SUCCESS; +} + +int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->idsTable); } + +int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; } + +#define NUM_OF_PAGES_ON_DISK(_r) ((_r)->numOfPages - (_r)->inMemPages) +#define FILE_SIZE_ON_DISK(_r) (NUM_OF_PAGES_ON_DISK(_r) * (_r)->pageSize) + +static int32_t createDiskResidesBuf(SDiskbasedResultBuf* pResultBuf) { + pResultBuf->fd = open(pResultBuf->path, O_CREAT | O_RDWR, 0666); + if (!FD_VALID(pResultBuf->fd)) { + qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + + assert(pResultBuf->numOfPages == pResultBuf->inMemPages); + pResultBuf->numOfPages += pResultBuf->incStep; + + int32_t ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize); + if (ret != TSDB_CODE_SUCCESS) { + qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + + pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0); + if (pResultBuf->pBuf == MAP_FAILED) { + qError("QInfo:%p failed to map temp file: %s. %s", pResultBuf->handle, pResultBuf->path, strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + + pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize; + return TSDB_CODE_SUCCESS; +} + +static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t incNumOfPages) { + assert(pResultBuf->numOfPages * pResultBuf->pageSize == pResultBuf->totalBufSize); + int32_t ret = TSDB_CODE_SUCCESS; + + if (pResultBuf->pBuf == NULL) { + assert(pResultBuf->fd == FD_INITIALIZER); + + if ((ret = createDiskResidesBuf(pResultBuf)) != TSDB_CODE_SUCCESS) { + return ret; + } + } else { + ret = munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf)); + pResultBuf->numOfPages += incNumOfPages; + + /* + * disk-based output buffer is exhausted, try to extend the disk-based buffer, the available disk space may + * be insufficient + */ + ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize); + if (ret != TSDB_CODE_SUCCESS) { + // dError("QInfo:%p failed to create intermediate result output file:%s. %s", pQInfo, pSupporter->extBufFile, + // strerror(errno)); + return TSDB_CODE_QRY_NO_DISKSPACE; + } + + pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize; + pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0); + + if (pResultBuf->pBuf == MAP_FAILED) { + // dError("QInfo:%p failed to map temp file: %s. %s", pQInfo, pSupporter->extBufFile, strerror(errno)); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + +#define NO_AVAILABLE_PAGES(_b) ((_b)->allocateId == (_b)->numOfPages - 1) + +static FORCE_INLINE int32_t getGroupIndex(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { + assert(pResultBuf != NULL); + + char* p = taosHashGet(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t)); + if (p == NULL) { // it is a new group id + return -1; + } + + int32_t slot = GET_INT32_VAL(p); + assert(slot >= 0 && slot < taosHashGetSize(pResultBuf->idsTable)); + + return slot; +} + +static int32_t addNewGroupId(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { + int32_t num = getNumOfResultBufGroupId(pResultBuf); // the num is the newest allocated group id slot + taosHashPut(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t), &num, sizeof(int32_t)); + + SArray* pa = taosArrayInit(1, sizeof(int32_t)); + taosArrayPush(pResultBuf->list, &pa); + + assert(taosArrayGetSize(pResultBuf->list) == taosHashGetSize(pResultBuf->idsTable)); + return num; +} + +static void registerPageId(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { + int32_t slot = getGroupIndex(pResultBuf, groupId); + if (slot < 0) { + slot = addNewGroupId(pResultBuf, groupId); + } + + SIDList pList = taosArrayGetP(pResultBuf->list, slot); + taosArrayPush(pList, &pageId); +} + +tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) { + if (NO_AVAILABLE_PAGES(pResultBuf)) { + if (extendDiskFileSize(pResultBuf, pResultBuf->incStep) != TSDB_CODE_SUCCESS) { + return NULL; + } + } + + // register new id in this group + *pageId = (++pResultBuf->allocateId); + registerPageId(pResultBuf, groupId, *pageId); + + // clear memory for the new page + tFilePage* page = getResBufPage(pResultBuf, *pageId); + memset(page, 0, pResultBuf->pageSize); + + return page; +} + +int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; } + +SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { + int32_t slot = getGroupIndex(pResultBuf, groupId); + if (slot < 0) { + return taosArrayInit(1, sizeof(int32_t)); + } else { + return taosArrayGetP(pResultBuf->list, slot); + } +} + +void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) { + if (pResultBuf == NULL) { + return; + } + + if (FD_VALID(pResultBuf->fd)) { + qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, file created:%s, file size:%d", handle, + pResultBuf->totalBufSize, pResultBuf->path, FILE_SIZE_ON_DISK(pResultBuf)); + + close(pResultBuf->fd); + munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf)); + pResultBuf->pBuf = NULL; + } else { + qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, no file created", handle, + pResultBuf->totalBufSize); + } + + unlink(pResultBuf->path); + tfree(pResultBuf->path); + + size_t size = taosArrayGetSize(pResultBuf->list); + for (int32_t i = 0; i < size; ++i) { + SArray* pa = taosArrayGetP(pResultBuf->list, i); + taosArrayDestroy(pa); + } + + taosArrayDestroy(pResultBuf->list); + taosHashCleanup(pResultBuf->idsTable); + + tfree(pResultBuf); +} + +int32_t getLastPageId(SIDList pList) { + size_t size = taosArrayGetSize(pList); + return *(int32_t*) taosArrayGet(pList, size - 1); +} + diff --git a/src/query/src/qUtil.c b/src/query/src/qutil.c similarity index 98% rename from src/query/src/qUtil.c rename to src/query/src/qutil.c index 4e71de830f..4c3595b0be 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qutil.c @@ -19,19 +19,17 @@ #include "taosmsg.h" #include "ttime.h" -#include "qExecutor.h" -#include "qUtil.h" +#include "qexecutor.h" +#include "qutil.h" int32_t getOutputInterResultBufSize(SQuery* pQuery) { int32_t size = 0; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - assert(pQuery->pSelectExpr[i].interBytes <= DEFAULT_INTERN_BUF_PAGE_SIZE); size += pQuery->pSelectExpr[i].interBytes; } assert(size > 0); - return size; } @@ -243,7 +241,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; memset(s, 0, size); - resetResultInfo(pResultInfo); + RESET_RESULT_INFO(pResultInfo); } pWindowRes->numOfRows = 0; diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index 9270c6e458..bf6f3bb50f 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -2,8 +2,8 @@ #include #include +#include "qresultbuf.h" #include "taos.h" -#include "qresultBuf.h" #include "tsdb.h" namespace { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 8463e791db..5cb0dffd46 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1376,7 +1376,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO * } */ - tsdbDebug("%p %d data blocks sort completed", pQueryHandle, cnt); + tsdbDebug("%p %d data blocks sort completed, %p", pQueryHandle, cnt, pQueryHandle->qinfo); cleanBlockOrderSupporter(&sup, numOfTables); free(pTree); diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 3764df4afc..7b73b1e17c 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -522,7 +522,7 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP void getTmpfilePath(const char *fileNamePrefix, char *dstPath) { const char* tdengineTmpFileNamePrefix = "tdengine-"; - char tmpPath[PATH_MAX] = {0}; + char tmpPath[PATH_MAX]; #ifdef WINDOWS char *tmpDir = getenv("tmp"); From 3cf5b7697b50a9986df2d8c4be27a3a809c65342 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 20 Jul 2020 00:02:51 +0800 Subject: [PATCH 09/10] [td-225] source files rename. --- src/client/inc/tscLocalMerge.h | 4 ++-- src/client/inc/tscUtil.h | 6 +++--- src/client/inc/tsclient.h | 4 ++-- src/client/src/tscFunctionImpl.c | 14 +++++++------- src/client/src/tscLocal.c | 12 ++++++------ src/client/src/tscSQLParser.c | 2 +- src/client/src/tscSql.c | 2 +- src/client/src/tscSubquery.c | 4 ++-- src/client/src/tscUtil.c | 6 +++--- src/query/inc/{qast.h => qAst.h} | 0 src/query/inc/{qexecutor.h => qExecutor.h} | 6 +++--- src/query/inc/{qextbuffer.h => qExtbuffer.h} | 2 +- src/query/inc/{qfill.h => qFill.h} | 2 +- src/query/inc/{qhistogram.h => qHistogram.h} | 0 src/query/inc/{qpercentile.h => qPercentile.h} | 2 +- src/query/inc/{qresultbuf.h => qResultbuf.h} | 4 ++-- ...qsyntaxtreefunction.h => qSyntaxtreefunction.h} | 0 src/query/inc/{qtsbuf.h => qTsbuf.h} | 0 src/query/inc/{qutil.h => qUtil.h} | 0 src/query/src/{qast.c => qAst.c} | 8 ++++---- src/query/src/{qexecutor.c => qExecutor.c} | 12 ++++++------ src/query/src/{qextbuffer.c => qExtbuffer.c} | 7 +++---- src/query/src/{qfill.c => qFill.c} | 4 ++-- src/query/src/{qfilterfunc.c => qFilterfunc.c} | 2 +- src/query/src/{qhistogram.c => qHistogram.c} | 2 +- src/query/src/{qparserImpl.c => qParserImpl.c} | 0 src/query/src/{qpercentile.c => qPercentile.c} | 6 +++--- src/query/src/{qresultbuf.c => qResultbuf.c} | 4 ++-- ...qsyntaxtreefunction.c => qSyntaxtreefunction.c} | 2 +- src/query/src/{qtokenizer.c => qTokenizer.c} | 0 src/query/src/{qtsbuf.c => qTsbuf.c} | 4 ++-- src/query/src/{qutil.c => qUtil.c} | 4 ++-- src/query/tests/astTest.cpp | 2 +- src/query/tests/histogramTest.cpp | 2 +- src/query/tests/resultBufferTest.cpp | 2 +- src/query/tests/tsBufTest.cpp | 2 +- 36 files changed, 66 insertions(+), 67 deletions(-) rename src/query/inc/{qast.h => qAst.h} (100%) rename src/query/inc/{qexecutor.h => qExecutor.h} (99%) rename src/query/inc/{qextbuffer.h => qExtbuffer.h} (98%) rename src/query/inc/{qfill.h => qFill.h} (99%) rename src/query/inc/{qhistogram.h => qHistogram.h} (100%) rename src/query/inc/{qpercentile.h => qPercentile.h} (98%) rename src/query/inc/{qresultbuf.h => qResultbuf.h} (99%) rename src/query/inc/{qsyntaxtreefunction.h => qSyntaxtreefunction.h} (100%) rename src/query/inc/{qtsbuf.h => qTsbuf.h} (100%) rename src/query/inc/{qutil.h => qUtil.h} (100%) rename src/query/src/{qast.c => qAst.c} (99%) rename src/query/src/{qexecutor.c => qExecutor.c} (99%) rename src/query/src/{qextbuffer.c => qExtbuffer.c} (99%) rename src/query/src/{qfill.c => qFill.c} (99%) rename src/query/src/{qfilterfunc.c => qFilterfunc.c} (99%) rename src/query/src/{qhistogram.c => qHistogram.c} (99%) rename src/query/src/{qparserImpl.c => qParserImpl.c} (100%) rename src/query/src/{qpercentile.c => qPercentile.c} (99%) rename src/query/src/{qresultbuf.c => qResultbuf.c} (99%) rename src/query/src/{qsyntaxtreefunction.c => qSyntaxtreefunction.c} (99%) rename src/query/src/{qtokenizer.c => qTokenizer.c} (100%) rename src/query/src/{qtsbuf.c => qTsbuf.c} (99%) rename src/query/src/{qutil.c => qUtil.c} (99%) diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index d38885ab2e..397a60d140 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -20,8 +20,8 @@ extern "C" { #endif -#include "qextbuffer.h" -#include "qfill.h" +#include "qExtbuffer.h" +#include "qFill.h" #include "taosmsg.h" #include "tlosertree.h" #include "tsclient.h" diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 786133a8f3..590f205e1d 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -23,11 +23,11 @@ extern "C" { /* * @date 2018/09/30 */ -#include "os.h" -#include "tbuffer.h" #include "exception.h" -#include "qextbuffer.h" +#include "os.h" +#include "qExtbuffer.h" #include "taosdef.h" +#include "tbuffer.h" #include "tscLocalMerge.h" #include "tsclient.h" diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 3f548c18ca..3b19a475dc 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -30,9 +30,9 @@ extern "C" { #include "tsqlfunction.h" #include "tutil.h" -#include "qexecutor.h" +#include "qExecutor.h" +#include "qTsbuf.h" #include "qsqlparser.h" -#include "qtsbuf.h" #include "tcmdtype.h" // forward declaration diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index fb15e4efd5..72ccd5adc6 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -14,15 +14,15 @@ */ #include "os.h" -#include "qextbuffer.h" -#include "qfill.h" -#include "qhistogram.h" -#include "qpercentile.h" -#include "qsyntaxtreefunction.h" -#include "qtsbuf.h" +#include "qAst.h" +#include "qExtbuffer.h" +#include "qFill.h" +#include "qHistogram.h" +#include "qPercentile.h" +#include "qSyntaxtreefunction.h" +#include "qTsbuf.h" #include "taosdef.h" #include "taosmsg.h" -#include "qast.h" #include "tscLog.h" #include "tscSubquery.h" #include "tscompression.h" diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 83700ce0a5..7f336daa91 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -16,14 +16,14 @@ #include "os.h" #include "taosmsg.h" -#include "tcache.h" -#include "tscUtil.h" -#include "tsclient.h" +#include "qExtbuffer.h" #include "taosdef.h" -#include "tscLog.h" -#include "qextbuffer.h" -#include "tschemautil.h" +#include "tcache.h" #include "tname.h" +#include "tscLog.h" +#include "tscUtil.h" +#include "tschemautil.h" +#include "tsclient.h" static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index b97e486449..09db65a105 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -18,9 +18,9 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "qAst.h" #include "taos.h" #include "taosmsg.h" -#include "qast.h" #include "tcompare.h" #include "tname.h" #include "tscLog.h" diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 6a14d3a65e..77584f168c 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -15,7 +15,7 @@ #include "hash.h" #include "os.h" -#include "qast.h" +#include "qAst.h" #include "tcache.h" #include "tnote.h" #include "trpc.h" diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 1dbc52efb0..5d26d09fae 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -14,8 +14,8 @@ */ #include "os.h" -#include "qtsbuf.h" -#include "qast.h" +#include "qAst.h" +#include "qTsbuf.h" #include "tcompare.h" #include "tscLog.h" #include "tscSubquery.h" diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 4e6133663e..27f6727339 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ -#include "os.h" -#include "hash.h" #include "tscUtil.h" +#include "hash.h" +#include "os.h" +#include "qAst.h" #include "taosmsg.h" -#include "qast.h" #include "tcache.h" #include "tkey.h" #include "tmd5.h" diff --git a/src/query/inc/qast.h b/src/query/inc/qAst.h similarity index 100% rename from src/query/inc/qast.h rename to src/query/inc/qAst.h diff --git a/src/query/inc/qexecutor.h b/src/query/inc/qExecutor.h similarity index 99% rename from src/query/inc/qexecutor.h rename to src/query/inc/qExecutor.h index 97a7565de8..127c38a6f8 100644 --- a/src/query/inc/qexecutor.h +++ b/src/query/inc/qExecutor.h @@ -18,10 +18,10 @@ #include "os.h" #include "hash.h" -#include "qfill.h" -#include "qresultbuf.h" +#include "qFill.h" +#include "qResultbuf.h" +#include "qTsbuf.h" #include "qsqlparser.h" -#include "qtsbuf.h" #include "query.h" #include "taosdef.h" #include "tarray.h" diff --git a/src/query/inc/qextbuffer.h b/src/query/inc/qExtbuffer.h similarity index 98% rename from src/query/inc/qextbuffer.h rename to src/query/inc/qExtbuffer.h index 2cbef2b1be..b57c48933f 100644 --- a/src/query/inc/qextbuffer.h +++ b/src/query/inc/qExtbuffer.h @@ -28,7 +28,7 @@ extern "C" { #include "tdataformat.h" #include "talgo.h" -#define DEFAULT_PAGE_SIZE (1024L*64) // 16k larger than the SHistoInfo +#define DEFAULT_PAGE_SIZE (1024L*4) // 16k larger than the SHistoInfo #define MAX_TMPFILE_PATH_LENGTH PATH_MAX #define INITIAL_ALLOCATION_BUFFER_SIZE 64 diff --git a/src/query/inc/qfill.h b/src/query/inc/qFill.h similarity index 99% rename from src/query/inc/qfill.h rename to src/query/inc/qFill.h index ee5974708a..db6a69c2c5 100644 --- a/src/query/inc/qfill.h +++ b/src/query/inc/qFill.h @@ -21,8 +21,8 @@ extern "C" { #endif #include "os.h" +#include "qExtbuffer.h" #include "taosdef.h" -#include "qextbuffer.h" typedef struct { STColumn col; // column info diff --git a/src/query/inc/qhistogram.h b/src/query/inc/qHistogram.h similarity index 100% rename from src/query/inc/qhistogram.h rename to src/query/inc/qHistogram.h diff --git a/src/query/inc/qpercentile.h b/src/query/inc/qPercentile.h similarity index 98% rename from src/query/inc/qpercentile.h rename to src/query/inc/qPercentile.h index c1227dad77..52f666c338 100644 --- a/src/query/inc/qpercentile.h +++ b/src/query/inc/qPercentile.h @@ -16,7 +16,7 @@ #ifndef TDENGINE_QPERCENTILE_H #define TDENGINE_QPERCENTILE_H -#include "qextbuffer.h" +#include "qExtbuffer.h" typedef struct MinMaxEntry { union { diff --git a/src/query/inc/qresultbuf.h b/src/query/inc/qResultbuf.h similarity index 99% rename from src/query/inc/qresultbuf.h rename to src/query/inc/qResultbuf.h index f901474cc1..41fccd177d 100644 --- a/src/query/inc/qresultbuf.h +++ b/src/query/inc/qResultbuf.h @@ -20,9 +20,9 @@ extern "C" { #endif -#include "os.h" -#include "qextbuffer.h" #include "hash.h" +#include "os.h" +#include "qExtbuffer.h" typedef struct SArray* SIDList; diff --git a/src/query/inc/qsyntaxtreefunction.h b/src/query/inc/qSyntaxtreefunction.h similarity index 100% rename from src/query/inc/qsyntaxtreefunction.h rename to src/query/inc/qSyntaxtreefunction.h diff --git a/src/query/inc/qtsbuf.h b/src/query/inc/qTsbuf.h similarity index 100% rename from src/query/inc/qtsbuf.h rename to src/query/inc/qTsbuf.h diff --git a/src/query/inc/qutil.h b/src/query/inc/qUtil.h similarity index 100% rename from src/query/inc/qutil.h rename to src/query/inc/qUtil.h diff --git a/src/query/src/qast.c b/src/query/src/qAst.c similarity index 99% rename from src/query/src/qast.c rename to src/query/src/qAst.c index 42f9f214f5..c2578c15c0 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qAst.c @@ -16,17 +16,17 @@ #include "os.h" -#include "tname.h" -#include "qast.h" -#include "tsdb.h" #include "exception.h" +#include "qAst.h" +#include "qSyntaxtreefunction.h" #include "qsqlparser.h" -#include "qsyntaxtreefunction.h" #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" #include "tbuffer.h" #include "tcompare.h" +#include "tname.h" +#include "tsdb.h" #include "tskiplist.h" #include "tsqlfunction.h" #include "tstoken.h" diff --git a/src/query/src/qexecutor.c b/src/query/src/qExecutor.c similarity index 99% rename from src/query/src/qexecutor.c rename to src/query/src/qExecutor.c index 9348f702d0..b4f825a7c2 100644 --- a/src/query/src/qexecutor.c +++ b/src/query/src/qExecutor.c @@ -13,19 +13,19 @@ * along with this program. If not, see . */ #include "os.h" +#include "qFill.h" +#include "taosmsg.h" #include "tcache.h" #include "tglobal.h" -#include "qfill.h" -#include "taosmsg.h" #include "exception.h" #include "hash.h" -#include "qast.h" -#include "qexecutor.h" -#include "qresultbuf.h" +#include "qAst.h" +#include "qExecutor.h" +#include "qResultbuf.h" +#include "qUtil.h" #include "query.h" #include "queryLog.h" -#include "qutil.h" #include "tlosertree.h" #include "tscompression.h" #include "ttime.h" diff --git a/src/query/src/qextbuffer.c b/src/query/src/qExtbuffer.c similarity index 99% rename from src/query/src/qextbuffer.c rename to src/query/src/qExtbuffer.c index afcf902123..69c5f0e24f 100644 --- a/src/query/src/qextbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -12,16 +12,15 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include "qExtbuffer.h" #include "os.h" -#include "tulog.h" -#include "qextbuffer.h" +#include "queryLog.h" #include "taos.h" #include "taosdef.h" #include "taosmsg.h" #include "tsqlfunction.h" -#include "ttime.h" +#include "tulog.h" #include "tutil.h" -#include "queryLog.h" #define COLMODEL_GET_VAL(data, schema, allrow, rowId, colId) \ (data + (schema)->pFields[colId].offset * (allrow) + (rowId) * (schema)->pFields[colId].field.bytes) diff --git a/src/query/src/qfill.c b/src/query/src/qFill.c similarity index 99% rename from src/query/src/qfill.c rename to src/query/src/qFill.c index 65951a5b9e..9dec2598bc 100644 --- a/src/query/src/qfill.c +++ b/src/query/src/qFill.c @@ -13,9 +13,9 @@ * along with this program. If not, see . */ +#include "qFill.h" #include "os.h" -#include "qfill.h" -#include "qextbuffer.h" +#include "qExtbuffer.h" #include "taosdef.h" #include "taosmsg.h" #include "tsqlfunction.h" diff --git a/src/query/src/qfilterfunc.c b/src/query/src/qFilterfunc.c similarity index 99% rename from src/query/src/qfilterfunc.c rename to src/query/src/qFilterfunc.c index 6f336434e1..7e9f5c7da5 100644 --- a/src/query/src/qfilterfunc.c +++ b/src/query/src/qFilterfunc.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "qexecutor.h" +#include "qExecutor.h" #include "taosmsg.h" #include "tcompare.h" #include "tsqlfunction.h" diff --git a/src/query/src/qhistogram.c b/src/query/src/qHistogram.c similarity index 99% rename from src/query/src/qhistogram.c rename to src/query/src/qHistogram.c index 26482e9f14..7835d82469 100644 --- a/src/query/src/qhistogram.c +++ b/src/query/src/qHistogram.c @@ -14,7 +14,7 @@ */ #include "os.h" -#include "qhistogram.h" +#include "qHistogram.h" #include "taosdef.h" #include "taosmsg.h" #include "tlosertree.h" diff --git a/src/query/src/qparserImpl.c b/src/query/src/qParserImpl.c similarity index 100% rename from src/query/src/qparserImpl.c rename to src/query/src/qParserImpl.c diff --git a/src/query/src/qpercentile.c b/src/query/src/qPercentile.c similarity index 99% rename from src/query/src/qpercentile.c rename to src/query/src/qPercentile.c index dc5ecb796a..c4490a01e7 100644 --- a/src/query/src/qpercentile.c +++ b/src/query/src/qPercentile.c @@ -13,12 +13,12 @@ * along with this program. If not, see . */ +#include "qPercentile.h" #include "os.h" -#include "tulog.h" -#include "qpercentile.h" +#include "queryLog.h" #include "taosdef.h" #include "taosmsg.h" -#include "queryLog.h" +#include "tulog.h" tExtMemBuffer *releaseBucketsExceptFor(tMemBucket *pMemBucket, int16_t segIdx, int16_t slotIdx) { tExtMemBuffer *pBuffer = NULL; diff --git a/src/query/src/qresultbuf.c b/src/query/src/qResultbuf.c similarity index 99% rename from src/query/src/qresultbuf.c rename to src/query/src/qResultbuf.c index 646d8b7986..f2194bbfb5 100644 --- a/src/query/src/qresultbuf.c +++ b/src/query/src/qResultbuf.c @@ -1,6 +1,6 @@ -#include "qresultbuf.h" +#include "qResultbuf.h" #include "hash.h" -#include "qextbuffer.h" +#include "qExtbuffer.h" #include "queryLog.h" #include "taoserror.h" diff --git a/src/query/src/qsyntaxtreefunction.c b/src/query/src/qSyntaxtreefunction.c similarity index 99% rename from src/query/src/qsyntaxtreefunction.c rename to src/query/src/qSyntaxtreefunction.c index 5719bb0188..2104edfd91 100644 --- a/src/query/src/qsyntaxtreefunction.c +++ b/src/query/src/qSyntaxtreefunction.c @@ -15,7 +15,7 @@ #include "os.h" -#include "qsyntaxtreefunction.h" +#include "qSyntaxtreefunction.h" #include "taosdef.h" #include "tutil.h" diff --git a/src/query/src/qtokenizer.c b/src/query/src/qTokenizer.c similarity index 100% rename from src/query/src/qtokenizer.c rename to src/query/src/qTokenizer.c diff --git a/src/query/src/qtsbuf.c b/src/query/src/qTsbuf.c similarity index 99% rename from src/query/src/qtsbuf.c rename to src/query/src/qTsbuf.c index b84fbded38..20b29107f5 100644 --- a/src/query/src/qtsbuf.c +++ b/src/query/src/qTsbuf.c @@ -1,7 +1,7 @@ -#include "qtsbuf.h" +#include "qTsbuf.h" +#include "taoserror.h" #include "tscompression.h" #include "tutil.h" -#include "taoserror.h" static int32_t getDataStartOffset(); static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInfo* pBlockInfo); diff --git a/src/query/src/qutil.c b/src/query/src/qUtil.c similarity index 99% rename from src/query/src/qutil.c rename to src/query/src/qUtil.c index 4c3595b0be..be84471493 100644 --- a/src/query/src/qutil.c +++ b/src/query/src/qUtil.c @@ -19,8 +19,8 @@ #include "taosmsg.h" #include "ttime.h" -#include "qexecutor.h" -#include "qutil.h" +#include "qExecutor.h" +#include "qUtil.h" int32_t getOutputInterResultBufSize(SQuery* pQuery) { int32_t size = 0; diff --git a/src/query/tests/astTest.cpp b/src/query/tests/astTest.cpp index df27087216..99f03a7ff8 100644 --- a/src/query/tests/astTest.cpp +++ b/src/query/tests/astTest.cpp @@ -3,8 +3,8 @@ #include #include +#include "qAst.h" #include "taosmsg.h" -#include "qast.h" #include "tsdb.h" #include "tskiplist.h" diff --git a/src/query/tests/histogramTest.cpp b/src/query/tests/histogramTest.cpp index c23f0f5924..4a5f7fbbbe 100644 --- a/src/query/tests/histogramTest.cpp +++ b/src/query/tests/histogramTest.cpp @@ -9,7 +9,7 @@ #include "tstoken.h" #include "tutil.h" -#include "qhistogram.h" +#include "qHistogram.h" /* test validate the names for table/database */ TEST(testCase, histogram_binary_search) { diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index bf6f3bb50f..f822deeb73 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -2,7 +2,7 @@ #include #include -#include "qresultbuf.h" +#include "qResultbuf.h" #include "taos.h" #include "tsdb.h" diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index f813261957..e9827518e1 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -5,10 +5,10 @@ #include "taos.h" #include "tsdb.h" +#include "qTsbuf.h" #include "tstoken.h" #include "ttime.h" #include "tutil.h" -#include "qtsbuf.h" namespace { /** From 731707608c9391392db8b95ab910f7f3fc6301d4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 20 Jul 2020 00:04:21 +0800 Subject: [PATCH 10/10] [td-225] fix bugs in descending traverse over in-men data --- src/tsdb/src/tsdbRead.c | 79 ++++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 5cb0dffd46..5b1afe3da8 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -21,7 +21,7 @@ #include "tcompare.h" #include "exception.h" -#include "../../../query/inc/qast.h" // todo move to common module +#include "../../query/inc/qAst.h" // todo move to common module #include "tlosertree.h" #include "tsdb.h" #include "tsdbMain.h" @@ -331,7 +331,8 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh tsdbDebug("%p uid:%" PRId64", tid:%d check data in mem from skey:%" PRId64 ", order:%d, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pHandle->qinfo); } else { - tsdbDebug("%p uid:%" PRId64 ", tid:%d no data in mem", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid); + tsdbDebug("%p uid:%"PRId64", tid:%d no data in mem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, + pHandle->qinfo); } if (!imemEmpty) { @@ -343,7 +344,8 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh tsdbDebug("%p uid:%" PRId64", tid:%d check data in imem from skey:%" PRId64 ", order:%d, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pHandle->qinfo); } else { - tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid); + tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, + pHandle->qinfo); } return true; @@ -354,7 +356,7 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { tSkipListDestroyIter(pCheckInfo->iiter); } -SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) { +SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { SDataRow rmem = NULL, rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); @@ -371,20 +373,35 @@ SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) { } if (rmem != NULL && rimem != NULL) { - if (dataRowKey(rmem) < dataRowKey(rimem)) { - pCheckInfo->chosen = 0; - return rmem; - } else if (dataRowKey(rmem) == dataRowKey(rimem)) { - // data ts are duplicated, ignore the data in mem + TSKEY r1 = dataRowKey(rmem); + TSKEY r2 = dataRowKey(rimem); + + if (r1 == r2) { // data ts are duplicated, ignore the data in mem tSkipListIterNext(pCheckInfo->iter); pCheckInfo->chosen = 1; return rimem; } else { - pCheckInfo->chosen = 1; - return rimem; + if (ASCENDING_TRAVERSE(order)) { + if (r1 < r2) { + pCheckInfo->chosen = 0; + return rmem; + } else { + pCheckInfo->chosen = 1; + return rimem; + } + } else { + if (r1 < r2) { + pCheckInfo->chosen = 1; + return rimem; + } else { + pCheckInfo->chosen = 0; + return rmem; + } + } } } + // at least one (rmem or rimem) is absent here if (rmem != NULL) { pCheckInfo->chosen = 0; return rmem; @@ -398,7 +415,7 @@ SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) { return NULL; } -static bool moveToNextRow(STableCheckInfo* pCheckInfo) { +static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { bool hasNext = false; if (pCheckInfo->chosen == 0) { if (pCheckInfo->iter != NULL) { @@ -412,19 +429,17 @@ static bool moveToNextRow(STableCheckInfo* pCheckInfo) { if (pCheckInfo->iiter != NULL) { return tSkipListIterGet(pCheckInfo->iiter) != NULL; } - } else { - if (pCheckInfo->chosen == 1) { - if (pCheckInfo->iiter != NULL) { - hasNext = tSkipListIterNext(pCheckInfo->iiter); - } + } else { //pCheckInfo->chosen == 1 + if (pCheckInfo->iiter != NULL) { + hasNext = tSkipListIterNext(pCheckInfo->iiter); + } - if (hasNext) { - return hasNext; - } + if (hasNext) { + return hasNext; + } - if (pCheckInfo->iter != NULL) { - return tSkipListIterGet(pCheckInfo->iter) != NULL; - } + if (pCheckInfo->iter != NULL) { + return tSkipListIterGet(pCheckInfo->iter) != NULL; } } @@ -445,7 +460,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { initTableMemIterator(pHandle, pCheckInfo); } - SDataRow row = getSDataRowInTableMem(pCheckInfo); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order); if (row == NULL) { return false; } @@ -650,7 +665,7 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo); - SDataRow row = getSDataRowInTableMem(pCheckInfo); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL; cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1); @@ -1033,7 +1048,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; do { - SDataRow row = getSDataRowInTableMem(pCheckInfo); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); if (row == NULL) { break; } @@ -1061,9 +1076,9 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* cur->lastKey = key + step; cur->mixBlock = true; - moveToNextRow(pCheckInfo); + moveToNextRowInMem(pCheckInfo); } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it - moveToNextRow(pCheckInfo); + moveToNextRowInMem(pCheckInfo); } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { if (cur->win.skey == TSKEY_INITIAL_VAL) { @@ -1072,7 +1087,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* int32_t end = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order); if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it - moveToNextRow(pCheckInfo); + moveToNextRowInMem(pCheckInfo); } int32_t start = -1; @@ -1754,7 +1769,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int STable* pTable = pCheckInfo->pTableObj; do { - SDataRow row = getSDataRowInTableMem(pCheckInfo); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); if (row == NULL) { break; } @@ -1775,11 +1790,11 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, pMeta, numOfCols, pTable); if (++numOfRows >= maxRowsToRead) { - moveToNextRow(pCheckInfo); + moveToNextRowInMem(pCheckInfo); break; } - } while(moveToNextRow(pCheckInfo)); + } while(moveToNextRowInMem(pCheckInfo)); assert(numOfRows <= maxRowsToRead);